From 06aa8c9117a6a8348093bf50df7148203e5de6f2 Mon Sep 17 00:00:00 2001 From: Jonathan Haas Date: Tue, 14 Jul 2026 08:00:38 -0700 Subject: [PATCH] chore: sync public mirror from internal --- .husky/pre-commit | 12 +- docs/AGENT_PROFILES.md | 6 + docs/CUSTOM_AGENTS.md | 46 ++ docs/PAINTER.md | 149 +++++++ .../plans/2026-07-14-agent-operations-ui.md | 67 +++ .../2026-07-14-ai-package-build-unblocker.md | 74 ++++ .../2026-07-14-governed-custom-agent-api.md | 75 ++++ .../plans/2026-07-14-oracle-rollout-gates.md | 71 ++++ .../2026-07-14-session-routing-receipts.md | 77 ++++ ...4-amp-agent-operations-adoptions-design.md | 108 +++++ evals/tools/surface-smoke-cases.json | 47 ++- packages/ai/tsconfig.build.json | 1 + packages/contracts/src/index.ts | 51 +++ packages/contracts/src/maestro-app-server.ts | 6 + packages/contracts/src/schemas.ts | 55 +++ .../desktop/src/renderer/lib/api-client.ts | 2 + .../components/agent-operations-tree.test.ts | 87 ++++ .../src/components/agent-operations-tree.ts | 92 ++++ ...mposer-trajectory-replay-lab-panel.test.ts | 61 +++ .../composer-trajectory-replay-lab-panel.ts | 120 +++++- packages/web/src/services/api-client.types.ts | 3 + scripts/env-reads-baseline.json | 11 + src/agent/index.ts | 45 ++ src/agent/oracle-consultation-policy.ts | 6 +- src/agent/oracle-policy-experiment.ts | 108 +++++ src/agent/oracle-policy-rollout.ts | 155 +++++++ src/agent/plugin-agent-api.ts | 40 ++ src/agent/plugin-agent-loader.ts | 112 +++++ src/agent/plugin-agent-registry.ts | 225 ++++++++++ src/agent/routing-receipt.ts | 95 +++++ src/agent/subagent-specs.ts | 4 +- src/agent/types.ts | 7 +- src/app-server/plugin-bundle-api.ts | 1 + src/cli-tui/commands/package-handlers.ts | 15 +- src/cli/args.ts | 5 +- src/cli/commands/painter.ts | 104 +++++ src/cli/help.ts | 3 + src/config/env-vars.ts | 14 + src/main.ts | 5 + src/packages/discovery.ts | 1 + src/packages/inspection.ts | 8 +- src/packages/loader.ts | 13 +- src/packages/runtime.ts | 11 +- src/packages/types.ts | 6 + src/server/handlers/chat-ws.ts | 96 ++++- src/server/handlers/chat.ts | 88 +++- src/server/handlers/package.ts | 1 + src/server/hosted-session-manager.ts | 33 ++ src/server/session-serialization.ts | 3 + src/server/sse-session.ts | 7 + src/services/image-providers/cost.ts | 208 +++++++++ src/services/image-providers/flux.ts | 143 +++++++ src/services/image-providers/index.ts | 53 +++ src/services/image-providers/masks.ts | 161 +++++++ src/services/image-providers/openai.ts | 151 +++++++ src/services/image-providers/types.ts | 74 ++++ src/services/intelligent-router/recorder.ts | 32 +- src/session/manager.ts | 26 ++ src/session/types.ts | 2 + src/telemetry/oracle-policy.ts | 21 + src/tools/index.ts | 4 + src/tools/lazy-registry.ts | 1 + src/tools/painter.ts | 393 ++++++++++++++++++ src/utils/terminal-image.ts | 127 ++++++ test/agent/oracle-consultation-policy.test.ts | 10 + test/agent/oracle-policy-experiment.test.ts | 80 ++++ test/agent/oracle-policy-rollout.test.ts | 111 +++++ test/agent/plugin-agent-loader.test.ts | 73 ++++ test/agent/plugin-agent-registry.test.ts | 146 +++++++ test/agent/routing-receipt.test.ts | 97 +++++ test/app-server/host-control-api.test.ts | 12 +- test/app-server/plugin-bundle-api.test.ts | 12 +- test/cli/painter.test.ts | 118 ++++++ test/packages/ai-tsconfig-boundary.test.ts | 14 + test/packages/maestro-packages.test.ts | 22 + test/services/image-providers/cost.test.ts | 250 +++++++++++ test/services/image-providers/flux.test.ts | 141 +++++++ test/services/image-providers/masks.test.ts | 138 ++++++ test/services/image-providers/openai.test.ts | 145 +++++++ test/services/intelligent-router.test.ts | 10 + test/session/session-manager.test.ts | 35 ++ test/telemetry/oracle-policy.test.ts | 35 ++ test/tools/painter.test.ts | 363 ++++++++++++++++ test/tools/tools.test.ts | 1 + test/utils/terminal-image.test.ts | 134 ++++++ test/web/chat-handler-profile.test.ts | 71 ++++ test/web/session-serialization.test.ts | 25 ++ test/web/sse-session.test.ts | 21 + 88 files changed, 5822 insertions(+), 70 deletions(-) create mode 100644 docs/CUSTOM_AGENTS.md create mode 100644 docs/PAINTER.md create mode 100644 docs/superpowers/plans/2026-07-14-agent-operations-ui.md create mode 100644 docs/superpowers/plans/2026-07-14-ai-package-build-unblocker.md create mode 100644 docs/superpowers/plans/2026-07-14-governed-custom-agent-api.md create mode 100644 docs/superpowers/plans/2026-07-14-oracle-rollout-gates.md create mode 100644 docs/superpowers/plans/2026-07-14-session-routing-receipts.md create mode 100644 docs/superpowers/specs/2026-07-14-amp-agent-operations-adoptions-design.md create mode 100644 packages/web/src/components/agent-operations-tree.test.ts create mode 100644 packages/web/src/components/agent-operations-tree.ts create mode 100644 src/agent/oracle-policy-experiment.ts create mode 100644 src/agent/oracle-policy-rollout.ts create mode 100644 src/agent/plugin-agent-api.ts create mode 100644 src/agent/plugin-agent-loader.ts create mode 100644 src/agent/plugin-agent-registry.ts create mode 100644 src/agent/routing-receipt.ts create mode 100644 src/cli/commands/painter.ts create mode 100644 src/services/image-providers/cost.ts create mode 100644 src/services/image-providers/flux.ts create mode 100644 src/services/image-providers/index.ts create mode 100644 src/services/image-providers/masks.ts create mode 100644 src/services/image-providers/openai.ts create mode 100644 src/services/image-providers/types.ts create mode 100644 src/telemetry/oracle-policy.ts create mode 100644 src/tools/painter.ts create mode 100644 src/utils/terminal-image.ts create mode 100644 test/agent/oracle-policy-experiment.test.ts create mode 100644 test/agent/oracle-policy-rollout.test.ts create mode 100644 test/agent/plugin-agent-loader.test.ts create mode 100644 test/agent/plugin-agent-registry.test.ts create mode 100644 test/agent/routing-receipt.test.ts create mode 100644 test/cli/painter.test.ts create mode 100644 test/packages/ai-tsconfig-boundary.test.ts create mode 100644 test/services/image-providers/cost.test.ts create mode 100644 test/services/image-providers/flux.test.ts create mode 100644 test/services/image-providers/masks.test.ts create mode 100644 test/services/image-providers/openai.test.ts create mode 100644 test/telemetry/oracle-policy.test.ts create mode 100644 test/tools/painter.test.ts create mode 100644 test/utils/terminal-image.test.ts diff --git a/.husky/pre-commit b/.husky/pre-commit index 1fcaca9c7..4fe133e87 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -12,13 +12,17 @@ fi # Run Composer Guardian (secrets + CI hygiene) before heavier tasks. bash "$ROOT/scripts/guardian.sh" --trigger pre-commit -mapfile -t STAGED_FORMAT_FILES < <( +# Biome config (biome.json files.include) scopes it to src/test/packages/docs, and +# Biome 1.9.x only handles ts/tsx/js/jsx/mjs/cjs/json/jsonc/css. Restrict the input +# to both, so a commit touching only out-of-scope files (e.g. scripts/*.json, docs/*.md) +# does not trip the "No files were processed" error and block the commit. +mapfile -t BIOME_FILES < <( git diff --cached --name-only --diff-filter=ACMR | - grep -E "\\.(ts|tsx|js|jsx|mjs|cjs|json|jsonc|css|md|yml|yaml)$" || true + grep -E "^(src|test|packages|docs)/.*\.(ts|tsx|js|jsx|mjs|cjs|json|jsonc|css)$" || true ) -if [ "${#STAGED_FORMAT_FILES[@]}" -gt 0 ]; then - bunx biome check "${STAGED_FORMAT_FILES[@]}" +if [ "${#BIOME_FILES[@]}" -gt 0 ]; then + bunx biome check "${BIOME_FILES[@]}" else echo "No staged Biome-supported files to check." fi diff --git a/docs/AGENT_PROFILES.md b/docs/AGENT_PROFILES.md index 7da45032e..ede60d3f4 100644 --- a/docs/AGENT_PROFILES.md +++ b/docs/AGENT_PROFILES.md @@ -50,3 +50,9 @@ Oracle configurations must be read-only and reasoning-capable at runtime. The bu An assistant response completing without an error is not a verified success. Automatic promotion requires at least 20 verified outcomes for the workload. Verification results, explicit user acceptance, rejection, and retries can supply outcome evidence; unverified runs remain useful for latency and cost observations only. Clients can request a profile through `X-Maestro-Agent-Profile` (or the compatibility `X-Composer-Agent-Profile`) and inspect the versioned resolved profile on the routing decision. + +## Outcome-calibrated Oracle experiments + +Hosts can enable deterministic session-level control/treatment assignment with `MAESTRO_ORACLE_EXPERIMENT_ID`, `MAESTRO_ORACLE_EXPERIMENT_ALLOCATION`, `MAESTRO_ORACLE_EXPERIMENT_CONTROL_VERSION`, and `MAESTRO_ORACLE_EXPERIMENT_TREATMENT_VERSION`. Configuration must be complete and allocation must be between `0` and `1`. + +The assigned arm and policy version appear in routing receipts and bounded telemetry attributes. Assignment is immutable for an experiment/session pair; promotion remains advisory and requires verified outcome gates. diff --git a/docs/CUSTOM_AGENTS.md b/docs/CUSTOM_AGENTS.md new file mode 100644 index 000000000..fe2a7508f --- /dev/null +++ b/docs/CUSTOM_AGENTS.md @@ -0,0 +1,46 @@ +# Governed Custom Agents + +Installed Maestro packages can prepare primary or subagent modes with the capability-scoped API exported from `src/agent/plugin-agent-api.ts`. + +Declare agent resource directories in the package manifest. Each child directory is one statically discoverable agent and can be filtered like other package resources: + +```json +{ + "keywords": ["maestro-package"], + "maestro": { "agents": ["./agents"] } +} +``` + +For example, `agents/focused-reviewer/agent.json` supplies the static `key`, `label`, and `entry` metadata. Runtime registration must match that metadata before the agent becomes available. + +```ts +const agents = createPluginAgentApi({ + policy: hostPolicy, + metadata: discoveredAgentMetadata, +}); + +const reviewer = agents.createAgent({ + key: "focused-reviewer", + label: "Focused reviewer", + description: "Reviews a bounded change", + systemPrompt: "Review the requested change and report actionable findings.", + model: "anthropic/claude-sonnet-4-6", + tools: ["read", "search"], + budgets: { maxTurns: 10, maxToolCalls: 20, maxCostUsd: 5 }, + approvalMode: "fail", + sandboxMode: "read-only", +}); + +agents.registerAgentMode({ + key: "focused-reviewer", + label: "Focused reviewer", + agent: reviewer, + primary: true, +}); +``` + +`tools: "all"` means every tool already allowed by the host policy, not every installed tool. Explicit tool lists, models, and all three budget limits must remain within the host policy. Approval and sandbox settings may be equal to or more restrictive than the host, never less restrictive. + +Registration is atomic and requires a matching static metadata declaration. Duplicate names, metadata mismatches, foreign or forged handles, unknown tools, disallowed models, invalid budgets, and permission escalation are rejected without modifying the registry. + +This API configures agent modes only. It does not add executable tools, UI extensions, raw process access, or authority beyond the package installation and host runtime policy. Extensions and MCP remain the supported executable-tool mechanisms. diff --git a/docs/PAINTER.md b/docs/PAINTER.md new file mode 100644 index 000000000..f58ffa681 --- /dev/null +++ b/docs/PAINTER.md @@ -0,0 +1,149 @@ +# Painter + +Painter generates and edits images from inside an agent thread. It is the +visual counterpart to the Oracle: where the Oracle brings a second model's +reasoning, Painter brings an image model's output. Use it for UI mockups, +app icons, illustrations, and editing existing images (for example, +redacting a screenshot before pasting it into a PR). + +Painter is a tool the main agent invokes. It is **not** always-on — tell the +agent to use it, the same way you would tell it to use the Oracle. + +## Setup + +Painter calls an image API and needs a credential in the environment where the +agent runs. The default provider is OpenAI (`gpt-image-2`): + +```bash +export OPENAI_API_KEY=sk-... +``` + +To use FLUX via fal.ai instead: + +```bash +export MAESTRO_PAINTER_PROVIDER=flux +export FAL_KEY= +``` + +## Providers + +| Provider | `MAESTRO_PAINTER_PROVIDER` | Generate | Edit | Mask | Credential | +| --- | --- | --- | --- | --- | --- | +| OpenAI (default) | `openai` | yes | yes | yes | `OPENAI_API_KEY` | +| FLUX (fal.ai) | `flux` | yes | no | no | `FAL_KEY` | + +Each provider declares a `supports` map; the painter checks it before calling +edit, so asking for an edit on FLUX fails fast with a clear error instead of a +buried API failure. + +## What it does + +- **Generate** — text prompt → new image. +- **Edit** — one to three input images + a prompt → edited image (OpenAI only). + Restrict changes to a region with a mask (see Masking). +- Outputs are **persisted to disk** and returned as absolute paths. The agent + references those paths in later turns. Transcripts stay small (image bytes + are never inlined as base64 into the conversation). + +## Masking + +Two ways to restrict an edit to part of an image: + +- **Bounding box** — pass `maskRegion: {x, y, width, height}` in pixels. Painter + synthesizes the mask PNG automatically (transparent = editable region, + matching OpenAI's mask semantics). Input dimensions come from the PNG header + (no deps), fall back to the optional `sharp` package, then an explicit + caller-supplied `maskSize`. Coordinates outside the image are clipped. +- **Mask file** — pass `mask` pointing at a PNG whose transparent pixels mark + the editable region. + +Omit both for a whole-image edit. + +## Output location + +Defaults to `~/.maestro/assets/painter/`. Override with +`MAESTRO_PAINTER_OUTPUT_DIR`. Files are named `painter--.png`. + +## Inline preview + +View a generated image inline in a capable terminal: + +```bash +maestro painter show ~/.maestro/assets/painter/painter-....png +``` + +Renders via iTerm2/WezTerm (OSC 1337) or kitty (graphics protocol). Run it from +a plain shell, not inside the full-screen TUI. This writes only to stdout — it +never routes through the agent loop, so it can't waste model tokens or corrupt +the conversation context. + +## Spend ceiling (opt-in) + +Painter checks a process-level budget before every API call. Enable by setting +both: + +- `MAESTRO_PAINTER_MAX_COST_CENTS` — whole-dollar-cents ceiling. +- `MAESTRO_PAINTER_PRICE_TABLE` — JSON mapping + `${model}|${size}|${quality}` → cents per image, with `*` wildcards, e.g. + `{"gpt-image-2|*|high": 8}`. + +When the next call would exceed the ceiling, Painter fails fast with a clear +reason and makes no API call. Prices are **not** fabricated: the default table +is empty, and when a ceiling is set but no price matches, the gate fails **open** +(allows the call) rather than blocking on missing data. + +## Configuration + +| Variable | Default | Purpose | +| --- | --- | --- | +| `OPENAI_API_KEY` / `FAL_KEY` | — | Required provider credential. | +| `MAESTRO_PAINTER_PROVIDER` | `openai` | `openai` or `flux`. | +| `MAESTRO_PAINTER_MODEL` | `gpt-image-2` (OpenAI) / `fal-ai/flux/schnell` (FLUX) | Model id. | +| `MAESTRO_PAINTER_BASE_URL` | — | OpenAI-compatible base URL (proxies). | +| `MAESTRO_PAINTER_OUTPUT_DIR` | `~/.maestro/assets/painter` | Where images are written. | +| `MAESTRO_PAINTER_TIMEOUT_MS` | `180000` | Per-call timeout. | +| `MAESTRO_PAINTER_MAX_COST_CENTS` | — | Opt-in spend ceiling (whole cents). | +| `MAESTRO_PAINTER_PRICE_TABLE` | — | JSON price table for the ceiling. | + +## Examples + +Ask the agent directly: + +- "Use the painter to create a UI mockup for the settings page, dark theme." +- "Use the painter to generate an app icon: dark background, glowing cyan cursor." +- "Redact the API keys in `screenshots/bug.png` using the painter." + +For edits, point the agent at one or more image paths (use `@`-mention). +Painter accepts up to three reference images. + +## Model notes + +`gpt-image-2` does **not** support transparency; requesting +`background: transparent` is rejected by the API. It supports arbitrary `WxH` +sizes where both dimensions are divisible by 16 and the aspect ratio is between +1:3 and 3:1, in addition to the standard sizes. + +## Permissions + +Painter writes files only inside the configured output directory. Inputs are +read-only — Painter never mutates a file you pass in. Painter is part of the +`advanced` tool category: available to `coder` and `custom` subagent types, and +explicitly denied from `explorer` (read-only), mirroring the Oracle. + +## Composing with Conductor (browser automation) + +The Maestro-native loop is Painter + Conductor together: + +1. **Mockup → diff.** Painter generates a UI mockup; Conductor screenshots the + live page; the agent compares and iterates. +2. **Bug → redact → PR.** Conductor captures a failing-state screenshot; + Painter redacts secrets; the agent opens a PR with the clean image. + +## Not yet implemented + +- Sixel rasterization (sixel terminals are detected but not encoded; needs a + real image library). +- FLUX editing / inpainting (fal exposes this on a separate endpoint). +- Inline rendering *inside* the full-screen TUI itself (the `maestro painter + show` CLI command covers plain-shell preview; a TUI-integrated viewer is a + separate piece of work in the TUI package). diff --git a/docs/superpowers/plans/2026-07-14-agent-operations-ui.md b/docs/superpowers/plans/2026-07-14-agent-operations-ui.md new file mode 100644 index 000000000..7cd6c6242 --- /dev/null +++ b/docs/superpowers/plans/2026-07-14-agent-operations-ui.md @@ -0,0 +1,67 @@ +# Agent Operations UI Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a web operations panel that presents durable parent/child agent lineage, live status, navigation, and safe cancellation. + +**Architecture:** Extend the replay-lab wire projection with complete lineage, derive a deterministic forest in a pure web helper, and render it in a focused panel embedded in the composer. Existing session navigation and abort paths remain the only control mechanisms. + +**Tech Stack:** TypeScript, Lit, Maestro trajectory/replay contracts, Vitest. + +## Global Constraints + +- Sparse legacy lineage renders as roots. +- No new execution-control channel or permission bypass. +- HTTP and persisted replay data remain compatible. + +--- + +### Task 1: Complete the web lineage contract + +**Files:** +- Modify: `packages/web/src/services/api-client.types.ts` +- Modify: `src/server/agent-trajectory-replay-lab.ts` +- Test: `test/server/agent-trajectory-replay-lab.test.ts` + +**Interfaces:** +- Produces: `TrajectoryReplayLabTimelineItem.parentAgentRunId?: string` and existing `childAgentRunId?: string` from server projection. + +- [ ] **Step 1: Add a failing projection test** asserting a parent and child timeline record retain both IDs. +- [ ] **Step 2: Run** `bunx vitest --run test/server/agent-trajectory-replay-lab.test.ts` and confirm the parent field assertion fails. +- [ ] **Step 3: Add `parentAgentRunId?: string` to the web type and copy the field in replay-lab projection.** +- [ ] **Step 4: Re-run the focused test** and expect PASS. +- [ ] **Step 5: Commit** with `git commit -m "feat(trajectory): project complete agent lineage"`. + +### Task 2: Derive a deterministic operations forest + +**Files:** +- Create: `packages/web/src/components/agent-operations-tree.ts` +- Create: `packages/web/src/components/agent-operations-tree.test.ts` + +**Interfaces:** +- Produces: `buildAgentOperationsTree(items): AgentOperationsNode[]` where each node contains `runId`, `parentRunId?`, `status`, `latestItem`, and `children`. + +- [ ] **Step 1: Write failing tests** for nested runs, missing parents, duplicate events, latest-status selection, and stable timestamp/run-ID ordering. +- [ ] **Step 2: Run** `bunx vitest --run packages/web/src/components/agent-operations-tree.test.ts` and confirm the missing module fails. +- [ ] **Step 3: Implement a pure two-pass builder:** coalesce items by run ID, attach known parents, retain orphans as roots, recursively sort by latest timestamp then run ID. +- [ ] **Step 4: Re-run the focused test** and expect PASS. +- [ ] **Step 5: Commit** with `git commit -m "feat(web): derive agent operations tree"`. + +### Task 3: Render and control the operations panel + +**Files:** +- Create: `packages/web/src/components/composer-agent-operations-panel.ts` +- Create: `packages/web/src/components/composer-agent-operations-panel.test.ts` +- Modify: `packages/web/src/components/composer-chat.ts` +- Modify: `packages/web/src/components/composer-chat-overlays.ts` + +**Interfaces:** +- Consumes: `ApiClient.getSessionReplayLab(sessionId)` and `buildAgentOperationsTree`. +- Emits: `open-session` with `{ sessionId, runId }`, `cancel-run` with `{ runId }`, and `close`. + +- [ ] **Step 1: Write failing component tests** that verify nesting, status copy, expansion, navigation events, and that cancellation is absent for terminal runs. +- [ ] **Step 2: Run** `bunx vitest --run packages/web/src/components/composer-agent-operations-panel.test.ts` and confirm the component is missing. +- [ ] **Step 3: Implement the panel and wire navigation/cancellation to existing composer handlers.** Disable cancellation while a request is pending and expose the server error inline. +- [ ] **Step 4: Run** the panel tests plus `packages/web/src/components/composer-chat.test.ts`; expect PASS. +- [ ] **Step 5: Build** with `npx nx run maestro-web:build --skip-nx-cache`; expect PASS. +- [ ] **Step 6: Commit** with `git commit -m "feat(web): add agent operations panel"`. diff --git a/docs/superpowers/plans/2026-07-14-ai-package-build-unblocker.md b/docs/superpowers/plans/2026-07-14-ai-package-build-unblocker.md new file mode 100644 index 000000000..cce571dd7 --- /dev/null +++ b/docs/superpowers/plans/2026-07-14-ai-package-build-unblocker.md @@ -0,0 +1,74 @@ +# AI Package Build Unblocker Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Restore the required Maestro test baseline by including painter image-provider dependencies in the AI package TypeScript project. + +**Architecture:** Keep the current AI facade boundary and explicitly include the root image-provider directory alongside the other temporary root-owned sources. Protect the include with a source-level regression test. + +**Tech Stack:** TypeScript project references, Vitest, Nx. + +## Global Constraints + +- Do not move painter implementation files in this unblocker. +- Do not skip hooks or required builds. + +--- + +### Task 1: Cover and repair the package input boundary + +**Files:** +- Modify: `packages/ai/tsconfig.build.json` +- Create: `test/packages/ai-tsconfig-boundary.test.ts` + +**Interfaces:** +- Consumes: `packages/ai/tsconfig.build.json#include`. +- Produces: inclusion of `../../src/services/image-providers/**/*.ts` in the AI composite project. + +- [ ] **Step 1: Write the failing test** + +```ts +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +describe("@evalops/ai TypeScript boundary", () => { + it("includes painter image-provider dependencies", () => { + const config = JSON.parse(readFileSync("packages/ai/tsconfig.build.json", "utf8")); + expect(config.include).toContain("../../src/services/image-providers/**/*.ts"); + }); +}); +``` + +- [ ] **Step 2: Verify the test and package build fail** + +Run: `bunx vitest --run test/packages/ai-tsconfig-boundary.test.ts` +Expected: FAIL because the include is absent. + +Run: `npx nx run @evalops/ai:build --skip-nx-cache` +Expected: FAIL with TS6307 for `src/services/image-providers/*.ts`. + +- [ ] **Step 3: Add the minimal include** + +```json +"../../src/services/image-providers/**/*.ts", +``` + +Place it next to the existing `../../src/tools/**/*.ts` facade input. + +- [ ] **Step 4: Verify the regression and baseline** + +Run: `bunx vitest --run test/packages/ai-tsconfig-boundary.test.ts` +Expected: PASS. + +Run: `npx nx run @evalops/ai:build --skip-nx-cache` +Expected: PASS. + +Run: `npx nx run maestro:test --skip-nx-cache` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add packages/ai/tsconfig.build.json test/packages/ai-tsconfig-boundary.test.ts +git commit -m "fix(ai): include painter providers in package build" +``` diff --git a/docs/superpowers/plans/2026-07-14-governed-custom-agent-api.md b/docs/superpowers/plans/2026-07-14-governed-custom-agent-api.md new file mode 100644 index 000000000..fd15d355e --- /dev/null +++ b/docs/superpowers/plans/2026-07-14-governed-custom-agent-api.md @@ -0,0 +1,75 @@ +# Governed Custom-Agent API Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let installed Maestro packages define policy-governed primary and subagent configurations through a typed API. + +**Architecture:** Packages statically declare agent metadata, then register matching runtime configurations with a capability-scoped registry. A validator intersects requested models, tools, budgets, and permissions with host policy before creating immutable agent handles. + +**Tech Stack:** TypeScript, Maestro packages, model/tool registries, permission profiles, Vitest. + +## Global Constraints + +- Registration cannot increase host authority. +- The first API configures agents but does not execute arbitrary tool implementations or UI code. +- Duplicate or mismatched registration fails atomically. + +--- + +### Task 1: Add package agent discovery + +**Files:** +- Modify: `src/packages/types.ts` +- Modify: `src/packages/loader.ts` +- Modify: `src/packages/runtime.ts` +- Modify: `src/app-server/plugin-bundle-api.ts` +- Modify: `packages/contracts/src/maestro-app-server.ts` +- Test: `test/packages/maestro-packages.test.ts` +- Test: `test/app-server/plugin-bundle-api.test.ts` + +**Interfaces:** +- Produces: scoped `agents` resource directories and statically discoverable `AgentModeMetadata { key, label, entry }`. + +- [ ] **Step 1: Write failing loader and app-server tests** for agent resources, scope precedence, and missing metadata. +- [ ] **Step 2: Run the focused tests** and confirm `agents` is absent. +- [ ] **Step 3: Extend package manifests/resources/runtime aggregation and plugin-bundle responses with `agents`, preserving existing filters and trust gates.** +- [ ] **Step 4: Re-run focused tests; expect PASS.** +- [ ] **Step 5: Commit** with `git commit -m "feat(packages): discover custom agents"`. + +### Task 2: Implement the governed registry API + +**Files:** +- Create: `src/agent/plugin-agent-registry.ts` +- Create: `src/agent/plugin-agent-api.ts` +- Modify: `src/agent/index.ts` +- Create: `test/agent/plugin-agent-registry.test.ts` + +**Interfaces:** +- Produces: `createAgent(config)`, `registerAgentMode(registration)`, `PluginAgentHandle`, and `PluginAgentPolicy`. + +- [ ] **Step 1: Write failing tests** for successful creation, primary-mode registration, duplicate keys, unknown tools, disallowed models, unbounded budgets, metadata mismatch, and permission escalation. +- [ ] **Step 2: Run** `bunx vitest --run test/agent/plugin-agent-registry.test.ts` and confirm missing module failure. +- [ ] **Step 3: Implement immutable handles and atomic validation.** `tools: "all"` means all tools already allowed by host policy; an explicit list is intersected with that same set. Budgets must be positive and no greater than host maxima. Requested approval/sandbox settings may only be equal or more restrictive. +- [ ] **Step 4: Re-run focused tests; expect PASS.** +- [ ] **Step 5: Commit** with `git commit -m "feat(agent): add governed custom-agent API"`. + +### Task 3: Load registrations and expose custom modes + +**Files:** +- Create: `src/agent/plugin-agent-loader.ts` +- Modify: `src/agent/modes.ts` +- Modify: `src/agent/subagent-specs.ts` +- Modify: `src/cli/commands/modes.ts` +- Create: `docs/CUSTOM_AGENTS.md` +- Create: `test/agent/plugin-agent-loader.test.ts` +- Modify: `test/cli/modes-command.test.ts` + +**Interfaces:** +- Consumes: installed package `agents` resources and the governed registry. +- Produces: custom modes in mode listing and custom agent handles usable by subagent dispatch. + +- [ ] **Step 1: Write failing loader and CLI tests** using a fixture package that declares and registers `focused-reviewer`. +- [ ] **Step 2: Run focused tests** and confirm the custom mode is absent. +- [ ] **Step 3: Load registrations in package trust order, expose only validated agents, and document the typed API plus rejection rules.** +- [ ] **Step 4: Run focused tests, package-boundary validation, and the Maestro build; expect PASS.** +- [ ] **Step 5: Commit** with `git commit -m "feat(agent): load package-defined agent modes"`. diff --git a/docs/superpowers/plans/2026-07-14-oracle-rollout-gates.md b/docs/superpowers/plans/2026-07-14-oracle-rollout-gates.md new file mode 100644 index 000000000..6fe8fedc5 --- /dev/null +++ b/docs/superpowers/plans/2026-07-14-oracle-rollout-gates.md @@ -0,0 +1,71 @@ +# Oracle Rollout Gates Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Evaluate Oracle policy candidates with deterministic experiments and verified-outcome promotion gates. + +**Architecture:** A pure assignment helper selects control or treatment by hashing experiment plus session. A pure evaluator compares verified aggregates against sample, quality, cost, latency, and safety thresholds and returns an advisory decision. + +**Tech Stack:** TypeScript, existing Oracle consultation policy, intelligent-router outcomes, trajectory telemetry, Vitest. + +## Global Constraints + +- Assistant completion alone is never verified success. +- At least 20 verified samples per arm are required. +- Evaluation is advisory and never mutates production policy. + +--- + +### Task 1: Deterministic experiment assignment + +**Files:** +- Create: `src/agent/oracle-policy-experiment.ts` +- Create: `test/agent/oracle-policy-experiment.test.ts` +- Modify: `src/services/intelligent-router/types.ts` + +**Interfaces:** +- Produces: `assignOraclePolicyExperiment({ experimentId, sessionId, allocation, controlVersion, treatmentVersion })` returning an immutable arm and policy version. + +- [ ] **Step 1: Write failing tests** for stability, experiment isolation, allocation boundaries, and control/treatment version projection. +- [ ] **Step 2: Run the focused test** and confirm missing module failure. +- [ ] **Step 3: Implement assignment with a stable SHA-256-derived bucket in `[0, 1)`.** Allocation `0` always yields control and `1` always yields treatment. +- [ ] **Step 4: Re-run focused tests; expect PASS.** +- [ ] **Step 5: Commit** with `git commit -m "feat(oracle): add deterministic policy experiments"`. + +### Task 2: Verified rollout evaluator + +**Files:** +- Create: `src/agent/oracle-policy-rollout.ts` +- Create: `test/agent/oracle-policy-rollout.test.ts` + +**Interfaces:** +- Produces: `evaluateOraclePolicyRollout(input): OraclePolicyRolloutDecision` with status `hold | promote | rollback`, sufficiency, metrics, and reasons. + +- [ ] **Step 1: Write failing table tests** for insufficient samples, mixed versions, safety rollback, quality regression, cost/latency ceiling failures, and eligible promotion. +- [ ] **Step 2: Run the focused test** and confirm missing module failure. +- [ ] **Step 3: Implement the pure evaluator.** Ignore unverified samples; require 20 verified samples per arm; rollback for any safety violation; hold when treatment success trails control by more than `0.05` or configured cost/latency ratios are exceeded; promote only when every gate passes. +- [ ] **Step 4: Re-run focused tests; expect PASS.** +- [ ] **Step 5: Commit** with `git commit -m "feat(oracle): gate rollout on verified outcomes"`. + +### Task 3: Apply and observe experiments + +**Files:** +- Modify: `src/agent/oracle-consultation-policy.ts` +- Modify: `src/services/intelligent-router/service.ts` +- Modify: `src/server/handlers/chat.ts` +- Modify: `src/server/handlers/chat-ws.ts` +- Modify: `src/telemetry/otel.ts` +- Modify: `docs/AGENT_PROFILES.md` +- Test: `test/agent/oracle-consultation-policy.test.ts` +- Test: `test/web/chat-handler-routing.test.ts` +- Test: `test/telemetry/telemetry-otel-routing.test.ts` + +**Interfaces:** +- Consumes: optional experiment configuration and verified routing outcomes. +- Produces: assignment in routing decisions/receipts and telemetry fields `oracle.experiment_id`, `oracle.arm`, and `oracle.policy_version`. + +- [ ] **Step 1: Write failing parity and telemetry tests** for identical HTTP/WebSocket assignment and receipt projection. +- [ ] **Step 2: Run focused tests** and confirm experiment fields are absent. +- [ ] **Step 3: Assign before Oracle policy evaluation, select the arm's policy version, project it to routing receipts, and emit bounded telemetry attributes.** +- [ ] **Step 4: Run focused tests, lint, evals, and full build; expect PASS.** +- [ ] **Step 5: Commit** with `git commit -m "feat(oracle): observe outcome-calibrated rollout"`. diff --git a/docs/superpowers/plans/2026-07-14-session-routing-receipts.md b/docs/superpowers/plans/2026-07-14-session-routing-receipts.md new file mode 100644 index 000000000..cdd253743 --- /dev/null +++ b/docs/superpowers/plans/2026-07-14-session-routing-receipts.md @@ -0,0 +1,77 @@ +# Session Routing Receipts Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Pin routing profiles per session and preserve a visible, immutable routing receipt on every assistant turn. + +**Architecture:** A pure resolver applies request, session, then compatibility-default precedence. Chat HTTP and WebSocket handlers emit the same contract, session metadata stores only the current pin, and each assistant message stores its own historical receipt. + +**Tech Stack:** TypeScript, `@evalops/contracts`, session JSONL persistence, SSE/WebSocket, Lit, Vitest. + +## Global Constraints + +- Changing a pin never rewrites prior receipts. +- HTTP and WebSocket routing semantics are identical. +- Legacy mode/model fields remain available. + +--- + +### Task 1: Define receipt and pin contracts + +**Files:** +- Modify: `packages/contracts/src/index.ts` +- Modify: `packages/contracts/src/schemas.ts` +- Modify: `src/session/types.ts` +- Create: `test/agent/routing-receipt.test.ts` +- Create: `src/agent/routing-receipt.ts` + +**Interfaces:** +- Produces: `AgentProfilePin`, `RoutingReceipt`, `RoutingReceiptSource`, and `createRoutingReceipt(decision, context)`. + +- [ ] **Step 1: Write failing tests** for source precedence and receipt projection including Oracle, fallback, and experiment fields. +- [ ] **Step 2: Run** `bunx vitest --run test/agent/routing-receipt.test.ts` and confirm missing exports fail. +- [ ] **Step 3: Implement the schemas and pure receipt builder.** Required receipt fields are `decisionId`, `requestedProfile`, `source`, `resolvedProfileId`, `resolvedProfileVersion`, `provider`, `model`, `reasoningEffort`, and `createdAt`; optional fields carry Oracle, fallback, and experiment detail. +- [ ] **Step 4: Re-run focused tests and build contracts; expect PASS.** +- [ ] **Step 5: Commit** with `git commit -m "feat(router): define per-turn routing receipts"`. + +### Task 2: Persist pins and receipts through both chat transports + +**Files:** +- Modify: `src/server/handlers/chat.ts` +- Modify: `src/server/handlers/chat-ws.ts` +- Modify: `src/server/handlers/sessions.ts` +- Modify: `src/server/hosted-session-manager.ts` +- Modify: `src/services/intelligent-router/types.ts` +- Test: `test/web/chat-handler-routing.test.ts` +- Test: `test/web/chat-ws.test.ts` +- Test: `test/session/session-manager.test.ts` + +**Interfaces:** +- Consumes: request `profileHint` and optional `persistProfile`. +- Produces: session metadata `agentProfilePin` and stream event `{ type: "routing_receipt", receipt }`. + +- [ ] **Step 1: Write failing precedence, immutability, and transport-parity tests.** +- [ ] **Step 2: Run the three focused suites** and confirm the new pin/receipt assertions fail. +- [ ] **Step 3: Resolve request → session → global fallback in one shared helper, validate before updating metadata, append the receipt to the assistant message, and stream it over SSE and WebSocket.** +- [ ] **Step 4: Re-run focused suites; expect PASS.** +- [ ] **Step 5: Commit** with `git commit -m "feat(router): pin profiles to sessions"`. + +### Task 3: Show profile control and receipts in composer + +**Files:** +- Modify: `packages/web/src/services/api-client.types.ts` +- Modify: `packages/web/src/services/api-client.ts` +- Create: `packages/web/src/components/composer-routing-receipt.ts` +- Create: `packages/web/src/components/composer-routing-receipt.test.ts` +- Modify: `packages/web/src/components/composer-chat-message-pane.ts` +- Modify: `packages/web/src/components/composer-chat.ts` + +**Interfaces:** +- Consumes: `RoutingReceipt` on assistant messages and the session pin update API. +- Produces: a compact receipt summary and expandable detail view. + +- [ ] **Step 1: Write failing tests** for summary copy, expanded details, fallback reason, and pin updates scoped to the active session. +- [ ] **Step 2: Run focused web tests** and confirm missing UI fails. +- [ ] **Step 3: Implement the receipt component and session-scoped profile selector.** +- [ ] **Step 4: Run focused tests and `npx nx run maestro-web:build --skip-nx-cache`; expect PASS.** +- [ ] **Step 5: Commit** with `git commit -m "feat(web): show session routing receipts"`. diff --git a/docs/superpowers/specs/2026-07-14-amp-agent-operations-adoptions-design.md b/docs/superpowers/specs/2026-07-14-amp-agent-operations-adoptions-design.md new file mode 100644 index 000000000..5fa12d63c --- /dev/null +++ b/docs/superpowers/specs/2026-07-14-amp-agent-operations-adoptions-design.md @@ -0,0 +1,108 @@ +# Amp-Inspired Agent Operations Adoptions + +## Objective + +Bring four proven Amp interaction patterns into Maestro without replacing Maestro's existing profile, trajectory, package, safety, or verified-outcome systems: + +1. An agent-operations UI for observing and controlling parent and child runs. +2. Session-pinned routing profiles with a visible receipt for every turn. +3. A governed custom-agent plugin API for primary modes and subagents. +4. Outcome-calibrated Oracle experiments and rollout gates. + +Each capability must be independently reviewable, mergeable, and releasable. A newly discovered `main` build regression in the painter package boundary must be fixed first because it prevents the repository's required baseline suite from running. + +## Product Decisions + +### Agent operations + +The web composer will expose an operations panel sourced from the existing durable trajectory and runtime-ledger data. It will group runs as a parent/child tree, show the latest status and activity summary, and let a user open the associated session or run. Controls will be limited to operations already authorized by Maestro's runtime, initially stop/cancel and navigation; the UI will not invent a second execution-control channel. + +The lineage contract will include both `parentAgentRunId` and `childAgentRunId` on web-facing timeline items. Tree construction will be deterministic, tolerate missing ancestors, and surface sparse or legacy records as roots rather than hiding them. + +### Session-pinned routing and receipts + +An agent profile selection will be stored per session. A request without an explicit override uses the session pin; an explicit request may update the pin only when the caller asks to persist it. The process-global mode remains a compatibility fallback for sessions that have no pin. + +Every completed or failed turn will expose a routing receipt containing: + +- requested profile and source (`request`, `session`, or compatibility default); +- resolved profile identifier and version; +- primary provider, model, and reasoning effort; +- Oracle policy decision; +- fallback or degradation reason, when applicable; +- experiment assignment, when applicable. + +The composer will render a compact receipt next to the assistant turn and provide an expanded detail view. Receipts are historical facts: changing the current pin does not rewrite earlier turns. + +### Governed custom-agent plugin API + +Maestro packages may register custom agents through a typed API modeled on `createAgent(config)` and `registerAgentMode(registration)`. Agents can be used as a main mode or spawned as subagents. Static package metadata declares discoverable agent modes before plugin execution and must match runtime registration. + +The runtime API is capability-scoped. Agent configuration may select only policy-approved models and tools, must provide bounded budgets, and cannot expand the host process's filesystem, network, approval, or sandbox authority. Registration fails closed for duplicate names, metadata mismatches, unknown tools, disallowed models, invalid budgets, or attempted permission escalation. Package trust and existing installation scope remain the outer trust boundary; an installed package does not automatically bypass runtime policy. + +The initial API deliberately excludes arbitrary UI extensions, raw process access, and dynamic tool implementation. Existing Maestro extensions and MCP remain the mechanisms for adding executable tools. This keeps the first agent API useful while preserving a reviewable security boundary. + +### Outcome-calibrated Oracle rollout + +Oracle policy evaluation will use deterministic control/treatment assignment keyed by experiment and session. Control uses the current policy; treatment uses the candidate policy. The assignment and policy version appear in routing receipts and trajectory telemetry. + +A rollout evaluator will accept only verified outcomes. Completion without verifier evidence remains unverified. Promotion requires all of the following: + +- at least 20 verified samples in each arm; +- treatment success rate is not worse than control beyond a five-percentage-point guardrail; +- treatment cost and latency stay within explicitly configured ceilings; +- no safety regression or policy violation; +- confidence and sample sufficiency are emitted with the decision. + +The evaluator returns `hold`, `promote`, or `rollback` plus machine-readable reasons. It never mutates production policy by itself; deployment or configuration automation consumes the decision through an explicit approval boundary. + +## Architecture and Data Flow + +The four capabilities extend existing boundaries: + +1. Runtime-ledger and trajectory events remain the source of lineage truth. +2. Session metadata owns the routing pin, and the chat handler resolves it before routing. +3. The routing decision produces an immutable receipt stored with turn/session history and streamed to clients. +4. Package loading discovers static agent declarations, then a constrained registry validates runtime registrations against policy. +5. Oracle experiment assignment is attached before policy evaluation; verified outcomes feed aggregate evaluation after the run. + +Contracts in `@evalops/contracts` will be the shared wire source for receipts, lineage, custom-agent registrations, and rollout decisions. HTTP and WebSocket paths must project the same contracts. + +## Error Handling + +- Missing lineage is represented explicitly and does not prevent other runs from rendering. +- Invalid or unavailable pinned profiles return a structured routing error and leave the prior valid pin unchanged. +- Receipt generation is required for routed turns; degradation details are recorded rather than silently discarded. +- Invalid custom-agent registrations fail atomically and do not partially modify the registry. +- Experiment evaluation rejects mixed policy versions, insufficient verified evidence, and invalid thresholds with machine-readable reasons. +- Existing compatibility clients continue to receive legacy model and mode fields. + +## Delivery Slices + +1. Fix the painter TypeScript package-boundary regression on `main`. +2. Land agent operations contracts, tree derivation, panel, navigation, and safe controls. +3. Land per-session profile pins and immutable per-turn routing receipts across HTTP, WebSocket, persistence, and web UI. +4. Land the governed custom-agent registry/API, static discovery metadata, package integration, and documentation. +5. Land deterministic Oracle experiments, verified-outcome gates, telemetry, and operator-facing decisions. +6. Mirror the merged internal commits through the existing public-sync automation and deploy one final image containing all slices to the internal environment. + +## Testing and Verification + +Every behavior change follows red-green-refactor development. Focused tests will cover lineage tree construction, legacy sparse records, session-pin precedence, receipt immutability, HTTP/WebSocket parity, plugin validation and escalation rejection, deterministic experiment assignment, and every rollout gate outcome. + +Before each PR, run the focused tests plus the affected package builds. Before final merge and deployment, run: + +- `bun run bun:lint` +- `npx nx run maestro:test --skip-nx-cache` +- `npx nx run maestro:evals --skip-nx-cache` +- `npx nx run maestro:build --skip-nx-cache` + +After deployment, verify the exact image digest, workload rollout, authenticated internal health, and the public mirror health workflow. + +## Non-Goals + +- Replacing Maestro's package, skill, extension, MCP, or sandbox systems. +- Giving plugins unrestricted host-process capabilities. +- Automatically promoting an Oracle policy without an explicit operator or deployment boundary. +- Building a new scheduler or distributed execution engine. +- Retrofitting every non-web client with the operations UI in this delivery. diff --git a/evals/tools/surface-smoke-cases.json b/evals/tools/surface-smoke-cases.json index ef7dd5e97..11bcc845a 100644 --- a/evals/tools/surface-smoke-cases.json +++ b/evals/tools/surface-smoke-cases.json @@ -4,11 +4,12 @@ "kind": "registry", "judgeRubric": "The coding tool bundle and the tool registry should expose the same expected default tools without missing critical coding entries.", "expected": { - "codingToolNames": [ - "read", - "list", - "oracle", - "find", + "codingToolNames": [ + "read", + "list", + "oracle", + "painter", + "find", "extract_document", "search", "parallel_ripgrep", @@ -34,22 +35,23 @@ "pipeline_log_activity" ], "registryNames": [ - "apply_patch", - "ask_user", - "background_tasks", - "bash", - "codesearch", - "diff", - "edit", - "extract_document", - "find", - "gh_issue", - "gh_pr", - "gh_repo", - "list", - "notebook_edit", - "oracle", - "parallel_ripgrep", + "apply_patch", + "ask_user", + "background_tasks", + "bash", + "codesearch", + "diff", + "edit", + "extract_document", + "find", + "gh_issue", + "gh_pr", + "gh_repo", + "list", + "notebook_edit", + "oracle", + "painter", + "parallel_ripgrep", "pipeline_create_signal", "pipeline_log_activity", "pipeline_search_contacts", @@ -69,7 +71,7 @@ "kind": "apiTools", "judgeRubric": "The API tools endpoint should enumerate the coding tools completely and report the correct coding count.", "expected": { - "codingCount": 27, + "codingCount": 28, "codingNames": [ "apply_patch", "ask_user", @@ -86,6 +88,7 @@ "list", "notebook_edit", "oracle", + "painter", "parallel_ripgrep", "pipeline_create_signal", "pipeline_log_activity", diff --git a/packages/ai/tsconfig.build.json b/packages/ai/tsconfig.build.json index b946aa7bc..df74ea006 100644 --- a/packages/ai/tsconfig.build.json +++ b/packages/ai/tsconfig.build.json @@ -12,6 +12,7 @@ "../../src/models/**/*.ts", "../../src/providers/**/*.ts", "../../src/tools/**/*.ts", + "../../src/services/image-providers/**/*.ts", "../../src/memory/backend.ts", "../../src/memory/platform-memory-client.ts", "../../src/memory/relevant-recall.ts", diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index ee50afeb1..7ce81a272 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -125,6 +125,50 @@ export type ComposerThinkingLevel = | "medium" | "high"; +export type RoutingReceiptSource = + | "request" + | "session" + | "compatibility_default"; + +export interface AgentProfilePin { + profile: string; + updatedAt: string; +} + +export interface RoutingReceiptOracle { + policyVersion: string; + mode: "available" | "recommended" | "required"; + reasons: readonly string[]; +} + +export interface RoutingReceiptFallback { + reason: string; + provider?: string; + model?: string; +} + +export interface RoutingReceiptExperiment { + experimentId: string; + arm: "control" | "treatment"; + policyVersion: string; +} + +/** Immutable historical model/profile routing facts for one assistant turn. */ +export interface RoutingReceipt { + decisionId: string; + requestedProfile: string; + source: RoutingReceiptSource; + resolvedProfileId: string; + resolvedProfileVersion: number; + provider: string; + model: string; + reasoningEffort: string; + createdAt: string; + oracle?: RoutingReceiptOracle; + fallback?: RoutingReceiptFallback; + experiment?: RoutingReceiptExperiment; +} + /** * Represents a tool invocation within a message. * @@ -204,6 +248,8 @@ export interface ComposerMessage { provider?: string; /** Original API for assistant messages (for cross-provider resume) */ api?: string; + /** Immutable record of the model/profile routing decision for this turn. */ + routingReceipt?: RoutingReceipt; /** Original model ID for assistant messages (for cross-provider resume) */ model?: string; } @@ -362,6 +408,10 @@ export interface ComposerFilesResponse { export interface ComposerChatRequest { /** Model ID to use (e.g., "claude-sonnet-4-5-20250929") */ model?: string; + /** Agent profile requested for this turn. */ + profile?: string; + /** Persist an explicitly requested profile as the session default. */ + persistProfile?: boolean; /** Conversation history including the current user message */ messages: ComposerMessage[]; /** Extended thinking level for supported models */ @@ -719,6 +769,7 @@ export interface ComposerToolRetryDecision { * Agent-level streaming events (SSE payloads). */ export type ComposerAgentEvent = + | { type: "routing_receipt"; receipt: RoutingReceipt } | { type: "agent_start" } | { type: "agent_end"; diff --git a/packages/contracts/src/maestro-app-server.ts b/packages/contracts/src/maestro-app-server.ts index e7dd1c81c..b34bafc7f 100644 --- a/packages/contracts/src/maestro-app-server.ts +++ b/packages/contracts/src/maestro-app-server.ts @@ -623,6 +623,12 @@ export type MaestroAppServerPluginBundle = Static< >; export const MaestroAppServerPluginBundleResourcesSchema = Type.Object({ + agents: Type.Optional( + Type.Object({ + user: Type.Array(Type.String()), + project: Type.Array(Type.String()), + }), + ), extensions: Type.Object({ user: Type.Array(Type.String()), project: Type.Array(Type.String()), diff --git a/packages/contracts/src/schemas.ts b/packages/contracts/src/schemas.ts index 553dc1f9f..411f117e1 100644 --- a/packages/contracts/src/schemas.ts +++ b/packages/contracts/src/schemas.ts @@ -48,6 +48,54 @@ export const ComposerThinkingLevelSchema = Type.Union([ Type.Literal("high"), ]); +export const RoutingReceiptSourceSchema = Type.Union([ + Type.Literal("request"), + Type.Literal("session"), + Type.Literal("compatibility_default"), +]); + +export const AgentProfilePinSchema = Type.Object({ + profile: Type.String({ minLength: 1 }), + updatedAt: Type.String({ minLength: 1 }), +}); + +export const RoutingReceiptOracleSchema = Type.Object({ + policyVersion: Type.String({ minLength: 1 }), + mode: Type.Union([ + Type.Literal("available"), + Type.Literal("recommended"), + Type.Literal("required"), + ]), + reasons: Type.Array(Type.String()), +}); + +export const RoutingReceiptFallbackSchema = Type.Object({ + reason: Type.String({ minLength: 1 }), + provider: Type.Optional(Type.String()), + model: Type.Optional(Type.String()), +}); + +export const RoutingReceiptExperimentSchema = Type.Object({ + experimentId: Type.String({ minLength: 1 }), + arm: Type.Union([Type.Literal("control"), Type.Literal("treatment")]), + policyVersion: Type.String({ minLength: 1 }), +}); + +export const RoutingReceiptSchema = Type.Object({ + decisionId: Type.String({ minLength: 1 }), + requestedProfile: Type.String({ minLength: 1 }), + source: RoutingReceiptSourceSchema, + resolvedProfileId: Type.String({ minLength: 1 }), + resolvedProfileVersion: Type.Number({ minimum: 1 }), + provider: Type.String({ minLength: 1 }), + model: Type.String({ minLength: 1 }), + reasoningEffort: Type.String({ minLength: 1 }), + createdAt: Type.String({ minLength: 1 }), + oracle: Type.Optional(RoutingReceiptOracleSchema), + fallback: Type.Optional(RoutingReceiptFallbackSchema), + experiment: Type.Optional(RoutingReceiptExperimentSchema), +}); + export const ComposerToolCallSchema = Type.Object({ name: Type.String(), status: Type.Union([ @@ -101,11 +149,14 @@ export const ComposerMessageSchema = Type.Object({ usage: Type.Optional(ComposerUsageSchema), provider: Type.Optional(Type.String()), api: Type.Optional(Type.String()), + routingReceipt: Type.Optional(RoutingReceiptSchema), model: Type.Optional(Type.String()), }); export const ComposerChatRequestSchema = Type.Object({ model: Type.Optional(Type.String()), + profile: Type.Optional(Type.String()), + persistProfile: Type.Optional(Type.Boolean()), messages: Type.Array(ComposerMessageSchema), thinkingLevel: Type.Optional(ComposerThinkingLevelSchema), sessionId: Type.Optional(Type.String()), @@ -574,6 +625,10 @@ export const ComposerSessionSchema = Type.Object({ }); export const ComposerAgentEventSchema = Type.Union([ + Type.Object({ + type: Type.Literal("routing_receipt"), + receipt: RoutingReceiptSchema, + }), Type.Object({ type: Type.Literal("agent_start") }), Type.Object({ type: Type.Literal("agent_end"), diff --git a/packages/desktop/src/renderer/lib/api-client.ts b/packages/desktop/src/renderer/lib/api-client.ts index 9bc047443..36f433106 100644 --- a/packages/desktop/src/renderer/lib/api-client.ts +++ b/packages/desktop/src/renderer/lib/api-client.ts @@ -227,6 +227,7 @@ export interface LspDetections { export type PackageScope = "local" | "project" | "user"; export interface PackageResourceFilters { + agents?: string[]; extensions?: string[]; skills?: string[]; prompts?: string[]; @@ -247,6 +248,7 @@ export interface PackageInspectionResult { errors: string[]; } | null; resources: { + agents?: string[]; extensions: string[]; skills: string[]; prompts: string[]; diff --git a/packages/web/src/components/agent-operations-tree.test.ts b/packages/web/src/components/agent-operations-tree.test.ts new file mode 100644 index 000000000..af9841430 --- /dev/null +++ b/packages/web/src/components/agent-operations-tree.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest"; +import type { TrajectoryReplayLabTimelineItem } from "../services/api-client.types.js"; +import { buildAgentOperationsTree } from "./agent-operations-tree.js"; + +function item( + id: string, + runId: string, + timestamp: string, + status: string, + overrides: Partial = {}, +): TrajectoryReplayLabTimelineItem { + return { + id, + timestamp, + type: "agent.progress", + title: id, + status, + source: "local", + visibility: "user", + agentRunId: runId, + ...overrides, + }; +} + +describe("buildAgentOperationsTree", () => { + it("builds parent and child runs with their latest status", () => { + const roots = buildAgentOperationsTree([ + item("parent-start", "parent", "2026-07-14T00:00:00.000Z", "running"), + item("child-spawn", "parent", "2026-07-14T00:00:01.000Z", "running", { + parentAgentRunId: "parent", + childAgentRunId: "child", + }), + item("child-done", "child", "2026-07-14T00:00:02.000Z", "completed", { + parentAgentRunId: "parent", + }), + ]); + + expect(roots).toHaveLength(1); + expect(roots[0]).toMatchObject({ + runId: "parent", + status: "running", + children: [ + { + runId: "child", + parentRunId: "parent", + status: "completed", + latestItem: { id: "child-done" }, + }, + ], + }); + }); + + it("keeps missing ancestors as roots and sorts deterministically", () => { + const roots = buildAgentOperationsTree([ + item("z-old", "z-run", "2026-07-14T00:00:00.000Z", "running", { + parentAgentRunId: "missing", + }), + item("a-latest", "a-run", "2026-07-14T00:00:03.000Z", "failed"), + item("z-latest", "z-run", "2026-07-14T00:00:03.000Z", "completed", { + parentAgentRunId: "missing", + }), + ]); + + expect(roots.map((node) => node.runId)).toEqual(["a-run", "z-run"]); + expect(roots[1]).toMatchObject({ + parentRunId: "missing", + status: "completed", + latestItem: { id: "z-latest" }, + }); + }); + + it("creates a child node from a spawn record before child progress arrives", () => { + const roots = buildAgentOperationsTree([ + item("spawn", "parent", "2026-07-14T00:00:00.000Z", "running", { + parentAgentRunId: "parent", + childAgentRunId: "child", + }), + ]); + + expect(roots[0]?.children[0]).toMatchObject({ + runId: "child", + parentRunId: "parent", + status: "running", + latestItem: { id: "spawn" }, + }); + }); +}); diff --git a/packages/web/src/components/agent-operations-tree.ts b/packages/web/src/components/agent-operations-tree.ts new file mode 100644 index 000000000..9ec84c772 --- /dev/null +++ b/packages/web/src/components/agent-operations-tree.ts @@ -0,0 +1,92 @@ +import type { TrajectoryReplayLabTimelineItem } from "../services/api-client.types.js"; + +export interface AgentOperationsNode { + runId: string; + parentRunId?: string; + status: string; + latestItem: TrajectoryReplayLabTimelineItem; + children: AgentOperationsNode[]; +} + +interface MutableAgentOperationsNode extends AgentOperationsNode { + children: MutableAgentOperationsNode[]; +} + +function isLater( + candidate: TrajectoryReplayLabTimelineItem, + current: TrajectoryReplayLabTimelineItem, +): boolean { + if (candidate.timestamp !== current.timestamp) { + return candidate.timestamp > current.timestamp; + } + return candidate.id > current.id; +} + +function updateNode( + nodes: Map, + runId: string, + item: TrajectoryReplayLabTimelineItem, + parentRunId?: string, +): MutableAgentOperationsNode { + const current = nodes.get(runId); + if (!current) { + const created: MutableAgentOperationsNode = { + runId, + ...(parentRunId && parentRunId !== runId ? { parentRunId } : {}), + status: item.status ?? "unknown", + latestItem: item, + children: [], + }; + nodes.set(runId, created); + return created; + } + if (!current.parentRunId && parentRunId && parentRunId !== runId) { + current.parentRunId = parentRunId; + } + if (isLater(item, current.latestItem)) { + current.latestItem = item; + current.status = item.status ?? "unknown"; + } + return current; +} + +function compareNodes( + left: MutableAgentOperationsNode, + right: MutableAgentOperationsNode, +): number { + const timestamp = right.latestItem.timestamp.localeCompare( + left.latestItem.timestamp, + ); + return timestamp || left.runId.localeCompare(right.runId); +} + +export function buildAgentOperationsTree( + items: readonly TrajectoryReplayLabTimelineItem[], +): AgentOperationsNode[] { + const nodes = new Map(); + for (const item of items) { + if (item.agentRunId) { + updateNode(nodes, item.agentRunId, item, item.parentAgentRunId); + } + if (item.childAgentRunId) { + updateNode( + nodes, + item.childAgentRunId, + item, + item.parentAgentRunId ?? item.agentRunId, + ); + } + } + + const roots: MutableAgentOperationsNode[] = []; + for (const node of nodes.values()) { + const parent = node.parentRunId ? nodes.get(node.parentRunId) : undefined; + if (parent && parent !== node) { + parent.children.push(node); + } else { + roots.push(node); + } + } + for (const node of nodes.values()) node.children.sort(compareNodes); + return roots.sort(compareNodes); +} diff --git a/packages/web/src/components/composer-trajectory-replay-lab-panel.test.ts b/packages/web/src/components/composer-trajectory-replay-lab-panel.test.ts index 8160b9493..4129f2fc8 100644 --- a/packages/web/src/components/composer-trajectory-replay-lab-panel.test.ts +++ b/packages/web/src/components/composer-trajectory-replay-lab-panel.test.ts @@ -69,6 +69,46 @@ describe("ComposerTrajectoryReplayLabPanel", () => { expect(getSessionReplayLab).not.toHaveBeenCalled(); }); + + it("renders agent lineage and emits run navigation", async () => { + const getSessionReplayLab = vi.fn().mockResolvedValue(replayLab()); + const apiClient = { getSessionReplayLab } as unknown as ApiClient; + const element = await fixture( + html``, + ); + await element.updateComplete; + await Promise.resolve(); + await element.updateComplete; + + const buttons = [...(element.shadowRoot?.querySelectorAll("button") ?? [])]; + buttons.find((button) => button.textContent?.trim() === "agents")?.click(); + await element.updateComplete; + + const text = element.shadowRoot?.textContent ?? ""; + expect(text).toContain("Agent operations"); + expect(text).toContain("parent-run"); + expect(text).toContain("child-run"); + expect(text).toContain("completed"); + + const opened = new Promise>((resolve) => { + element.addEventListener("open-agent-run", (event) => + resolve(event as CustomEvent<{ runId: string }>), + ); + }); + const childButton = [ + ...(element.shadowRoot?.querySelectorAll( + "button[data-run-id]", + ) ?? []), + ].find((button) => button.dataset.runId === "child-run"); + childButton?.click(); + expect((await opened).detail).toEqual({ runId: "child-run" }); + await element.updateComplete; + expect(element.shadowRoot?.textContent ?? "").toContain("Selected run"); + expect(element.shadowRoot?.textContent ?? "").toContain("Child completed"); + }); }); function replayLab() { @@ -97,6 +137,27 @@ function replayLab() { }, timeline: { items: [ + { + id: "agent-parent", + timestamp: "2026-05-18T00:00:00.000Z", + type: "agent.started", + title: "Parent started", + status: "running", + visibility: "user", + source: "local", + agentRunId: "parent-run", + }, + { + id: "agent-child", + timestamp: "2026-05-18T00:00:01.000Z", + type: "agent.completed", + title: "Child completed", + status: "completed", + visibility: "user", + source: "local", + agentRunId: "child-run", + parentAgentRunId: "parent-run", + }, { id: "tool-result:call-read", timestamp: "2026-05-18T00:00:02.000Z", diff --git a/packages/web/src/components/composer-trajectory-replay-lab-panel.ts b/packages/web/src/components/composer-trajectory-replay-lab-panel.ts index 06b587f4e..7e1dae2c1 100644 --- a/packages/web/src/components/composer-trajectory-replay-lab-panel.ts +++ b/packages/web/src/components/composer-trajectory-replay-lab-panel.ts @@ -8,8 +8,12 @@ import type { TrajectoryReplayLabResponse, TrajectoryReplayLabTimelineItem, } from "../services/api-client.js"; +import { + type AgentOperationsNode, + buildAgentOperationsTree, +} from "./agent-operations-tree.js"; -type ReplayLabView = "trajectory" | "replay" | "score"; +type ReplayLabView = "agents" | "trajectory" | "replay" | "score"; function formatTimestamp(timestamp: string): string { const date = new Date(timestamp); @@ -33,6 +37,19 @@ function toneForStatus(status: string | undefined): string { return "normal"; } +function findAgentNode( + nodes: readonly AgentOperationsNode[], + runId: string | null, +): AgentOperationsNode | undefined { + if (!runId) return undefined; + for (const node of nodes) { + if (node.runId === runId) return node; + const child = findAgentNode(node.children, runId); + if (child) return child; + } + return undefined; +} + @customElement("composer-trajectory-replay-lab-panel") export class ComposerTrajectoryReplayLabPanel extends LitElement { static override styles = css` @@ -220,6 +237,33 @@ export class ComposerTrajectoryReplayLabPanel extends LitElement { overflow-wrap: anywhere; } + .agent-node { + display: grid; + gap: 0.35rem; + } + + .agent-children { + display: grid; + gap: 0.35rem; + margin-left: 0.85rem; + padding-left: 0.55rem; + border-left: 1px solid var(--border-primary, #1e2023); + } + + .agent-run { + width: 100%; + height: auto; + padding: 0.55rem 0.6rem; + text-align: left; + display: flex; + justify-content: space-between; + gap: 0.5rem; + } + + .agent-run-id { + overflow-wrap: anywhere; + } + .empty, .error, .loading { @@ -251,6 +295,7 @@ export class ComposerTrajectoryReplayLabPanel extends LitElement { @state() private loading = false; @state() private error: string | null = null; @state() private view: ReplayLabView = "trajectory"; + @state() private selectedRunId: string | null = null; private requestId = 0; @@ -327,7 +372,7 @@ export class ComposerTrajectoryReplayLabPanel extends LitElement { } private renderTabs() { - const tabs: ReplayLabView[] = ["trajectory", "replay", "score"]; + const tabs: ReplayLabView[] = ["agents", "trajectory", "replay", "score"]; return html`
${tabs.map( (tab) => html` + ${ + node.children.length + ? html`
+ ${node.children.map((child) => this.renderAgentNode(child))} +
` + : "" + } +
`; + } + + private renderAgents(lab: TrajectoryReplayLabResponse) { + const roots = buildAgentOperationsTree(lab.timeline.items); + const selected = findAgentNode(roots, this.selectedRunId); + return html`
+
Agent operations
+ ${ + roots.length + ? roots.map((node) => this.renderAgentNode(node)) + : html`
No agent runs found.
` + } + ${ + selected + ? html`
+
Selected run
+
+
${selected.latestItem.title}
+
${selected.status}
+
+
+ ${selected.runId}
${selected.latestItem.type} · + ${formatTimestamp(selected.latestItem.timestamp)} +
+
` + : "" + } +
`; + } + private renderTimelineItem(item: TrajectoryReplayLabTimelineItem) { return html`
@@ -485,11 +589,13 @@ export class ComposerTrajectoryReplayLabPanel extends LitElement { ${this.renderTabs()}
${ - this.view === "trajectory" - ? this.renderTrajectory(lab) - : this.view === "replay" - ? this.renderReplay(lab) - : this.renderScore(lab) + this.view === "agents" + ? this.renderAgents(lab) + : this.view === "trajectory" + ? this.renderTrajectory(lab) + : this.view === "replay" + ? this.renderReplay(lab) + : this.renderScore(lab) }
`; diff --git a/packages/web/src/services/api-client.types.ts b/packages/web/src/services/api-client.types.ts index 8b5ea9b52..9450dd798 100644 --- a/packages/web/src/services/api-client.types.ts +++ b/packages/web/src/services/api-client.types.ts @@ -159,6 +159,7 @@ export interface AttachmentTextExtractionResponse { export type PackageScope = "local" | "project" | "user"; export interface PackageResourceFilters { + agents?: string[]; extensions?: string[]; skills?: string[]; prompts?: string[]; @@ -179,6 +180,7 @@ export interface PackageInspectionResult { errors: string[]; } | null; resources: { + agents?: string[]; extensions: string[]; skills: string[]; prompts: string[]; @@ -441,6 +443,7 @@ export interface TrajectoryReplayLabTimelineItem { pendingRequestId?: string; artifactId?: string; agentRunId?: string; + parentAgentRunId?: string; childAgentRunId?: string; } diff --git a/scripts/env-reads-baseline.json b/scripts/env-reads-baseline.json index 84dac6c90..16c466f48 100644 --- a/scripts/env-reads-baseline.json +++ b/scripts/env-reads-baseline.json @@ -725,6 +725,10 @@ "MAESTRO_RATE_LIMIT_SESSION", "MAESTRO_RATE_LIMIT_WINDOW_MS" ], + "src/services/image-providers/cost.ts": [ + "MAESTRO_PAINTER_MAX_COST_CENTS", + "MAESTRO_PAINTER_PRICE_TABLE" + ], "src/session/file-writer.ts": [ "NODE_ENV", "VITEST" @@ -823,6 +827,13 @@ "MAESTRO_EVALOPS_ACCESS_TOKEN", "MAESTRO_ORACLE_MODEL" ], + "src/tools/painter.ts": [ + "MAESTRO_PAINTER_BASE_URL", + "MAESTRO_PAINTER_MODEL", + "MAESTRO_PAINTER_OUTPUT_DIR", + "MAESTRO_PAINTER_PROVIDER", + "MAESTRO_PAINTER_TIMEOUT_MS" + ], "src/tools/pipeline.ts": [ "PIPELINE_API_MAX_ATTEMPTS", "PIPELINE_API_TIMEOUT_MS", diff --git a/src/agent/index.ts b/src/agent/index.ts index 310b7d700..6b135faf1 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -121,6 +121,51 @@ export { parseAgentProfileLevel, resolveAgentProfile, } from "./profiles.js"; +export { + createPluginAgentApi, + type PluginAgentApi, + type PluginAgentBudgets, + type PluginAgentConfig, + type PluginAgentHandle, + type PluginAgentModeMetadata, + type PluginAgentModeRegistration, + type PluginAgentPolicy, + type PluginAgentSandboxMode, + type RegisteredPluginAgentMode, +} from "./plugin-agent-api.js"; +export { + loadConfiguredPluginAgentMetadata, + loadPluginAgentMetadataFromDirectories, + type LoadedPluginAgentModeMetadata, + type PluginAgentMetadataLoadResult, +} from "./plugin-agent-loader.js"; +export { + createRoutingReceipt, + resolveAgentProfileSelection, + type AgentProfileSelection, + type AgentProfileSelectionInput, + type RoutingReceiptContext, + type RoutingReceiptDecision, +} from "./routing-receipt.js"; +export { + assignOraclePolicyExperiment, + assignConfiguredOraclePolicyExperiment, + getOraclePolicyExperimentConfig, + type OraclePolicyExperimentArm, + type OraclePolicyExperimentAssignment, + type OraclePolicyExperimentConfig, + type OraclePolicyExperimentInput, +} from "./oracle-policy-experiment.js"; +export { + DEFAULT_ORACLE_ROLLOUT_MAX_SUCCESS_RATE_REGRESSION, + DEFAULT_ORACLE_ROLLOUT_MIN_VERIFIED_SAMPLES, + evaluateOraclePolicyRollout, + type OraclePolicyRolloutDecision, + type OraclePolicyRolloutInput, + type OraclePolicyRolloutMetrics, + type OraclePolicyRolloutStatus, + type OraclePolicyVerifiedAggregate, +} from "./oracle-policy-rollout.js"; export { loadAgentProfiles, loadAgentProfilesFromDirectory, diff --git a/src/agent/oracle-consultation-policy.ts b/src/agent/oracle-consultation-policy.ts index 4122b526d..458f1d534 100644 --- a/src/agent/oracle-consultation-policy.ts +++ b/src/agent/oracle-consultation-policy.ts @@ -6,6 +6,7 @@ export const ORACLE_CONSULTATION_POLICY_VERSION = export type OracleConsultationMode = "available" | "recommended" | "required"; export interface OracleConsultationPolicyInput { + policyVersion?: string; profileLevel: AgentProfileLevel; taskType: string; taskSummary?: string; @@ -13,7 +14,7 @@ export interface OracleConsultationPolicyInput { } export interface OracleConsultationDecision { - policyVersion: typeof ORACLE_CONSULTATION_POLICY_VERSION; + policyVersion: string; evalSuite: "oracle-consultation-policy-v1"; mode: OracleConsultationMode; reasons: string[]; @@ -75,7 +76,8 @@ export function recommendOracleConsultation( if (reasons.length === 0) reasons.push("oracle_available_on_demand"); return { - policyVersion: ORACLE_CONSULTATION_POLICY_VERSION, + policyVersion: + input.policyVersion?.trim() || ORACLE_CONSULTATION_POLICY_VERSION, evalSuite: "oracle-consultation-policy-v1", mode, reasons, diff --git a/src/agent/oracle-policy-experiment.ts b/src/agent/oracle-policy-experiment.ts new file mode 100644 index 000000000..94d398763 --- /dev/null +++ b/src/agent/oracle-policy-experiment.ts @@ -0,0 +1,108 @@ +import { createHash } from "node:crypto"; +import { ENV_VARS, getEnvString } from "../config/env-vars.js"; + +export type OraclePolicyExperimentArm = "control" | "treatment"; + +export interface OraclePolicyExperimentInput { + experimentId: string; + sessionId: string; + /** Fraction of sessions assigned to treatment, in the inclusive range 0..1. */ + allocation: number; + controlVersion: string; + treatmentVersion: string; +} + +export interface OraclePolicyExperimentAssignment { + readonly experimentId: string; + readonly arm: OraclePolicyExperimentArm; + readonly policyVersion: string; + readonly bucket: number; +} + +export interface OraclePolicyExperimentConfig { + experimentId: string; + allocation: number; + controlVersion: string; + treatmentVersion: string; +} + +const BUCKET_BYTES = 6; +const BUCKET_SPACE = 2 ** (BUCKET_BYTES * 8); + +/** Deterministically assign a session to one immutable Oracle policy arm. */ +export function assignOraclePolicyExperiment( + input: OraclePolicyExperimentInput, +): OraclePolicyExperimentAssignment { + if ( + !Number.isFinite(input.allocation) || + input.allocation < 0 || + input.allocation > 1 + ) { + throw new RangeError( + "Oracle experiment allocation must be between 0 and 1", + ); + } + + const digest = createHash("sha256") + .update(input.experimentId) + .update("\0") + .update(input.sessionId) + .digest(); + const bucket = digest.readUIntBE(0, BUCKET_BYTES) / BUCKET_SPACE; + const arm: OraclePolicyExperimentArm = + input.allocation === 1 || + (input.allocation !== 0 && bucket < input.allocation) + ? "treatment" + : "control"; + + return Object.freeze({ + experimentId: input.experimentId, + arm, + policyVersion: + arm === "treatment" ? input.treatmentVersion : input.controlVersion, + bucket, + }); +} + +/** Resolve the optional host-configured experiment. Partial configs fail closed. */ +export function getOraclePolicyExperimentConfig(): + | OraclePolicyExperimentConfig + | undefined { + const experimentId = getEnvString(ENV_VARS.ORACLE_EXPERIMENT_ID); + if (!experimentId) return undefined; + const allocationValue = getEnvString(ENV_VARS.ORACLE_EXPERIMENT_ALLOCATION); + const controlVersion = getEnvString( + ENV_VARS.ORACLE_EXPERIMENT_CONTROL_VERSION, + ); + const treatmentVersion = getEnvString( + ENV_VARS.ORACLE_EXPERIMENT_TREATMENT_VERSION, + ); + const allocation = Number(allocationValue); + if ( + !allocationValue || + !Number.isFinite(allocation) || + allocation < 0 || + allocation > 1 || + !controlVersion || + !treatmentVersion + ) { + throw new Error( + "Oracle experiment requires allocation in 0..1 plus control and treatment versions", + ); + } + return Object.freeze({ + experimentId, + allocation, + controlVersion, + treatmentVersion, + }); +} + +export function assignConfiguredOraclePolicyExperiment( + sessionId: string, +): OraclePolicyExperimentAssignment | undefined { + const config = getOraclePolicyExperimentConfig(); + return config + ? assignOraclePolicyExperiment({ ...config, sessionId }) + : undefined; +} diff --git a/src/agent/oracle-policy-rollout.ts b/src/agent/oracle-policy-rollout.ts new file mode 100644 index 000000000..b70f2e771 --- /dev/null +++ b/src/agent/oracle-policy-rollout.ts @@ -0,0 +1,155 @@ +export type OraclePolicyRolloutStatus = "hold" | "promote" | "rollback"; + +export interface OraclePolicyVerifiedAggregate { + policyVersion: string; + verifiedSamples: number; + successes: number; + averageCostUsd: number; + averageLatencyMs: number; + safetyViolations: number; +} + +export interface OraclePolicyRolloutInput { + experimentId: string; + expectedControlVersion: string; + expectedTreatmentVersion: string; + control: OraclePolicyVerifiedAggregate; + treatment: OraclePolicyVerifiedAggregate; + minVerifiedSamples?: number; + maxSuccessRateRegression?: number; + maxCostRatio?: number; + maxLatencyRatio?: number; +} + +export interface OraclePolicyRolloutMetrics { + controlSuccessRate: number; + treatmentSuccessRate: number; + successRateDelta: number; + costRatio: number; + latencyRatio: number; +} + +export interface OraclePolicyRolloutDecision { + readonly experimentId: string; + readonly status: OraclePolicyRolloutStatus; + readonly sufficient: boolean; + readonly reasons: readonly string[]; + readonly metrics: Readonly; +} + +export const DEFAULT_ORACLE_ROLLOUT_MIN_VERIFIED_SAMPLES = 20; +export const DEFAULT_ORACLE_ROLLOUT_MAX_SUCCESS_RATE_REGRESSION = 0.05; + +function rate(numerator: number, denominator: number): number { + return denominator > 0 ? numerator / denominator : 0; +} + +function ratio(treatment: number, control: number): number { + if (control > 0) return treatment / control; + return treatment > 0 ? Number.POSITIVE_INFINITY : 1; +} + +/** + * Evaluate an Oracle policy experiment from verified outcome aggregates. + * This function is advisory: it never mutates or activates a production policy. + */ +export function evaluateOraclePolicyRollout( + input: OraclePolicyRolloutInput, +): OraclePolicyRolloutDecision { + const minVerifiedSamples = + input.minVerifiedSamples ?? DEFAULT_ORACLE_ROLLOUT_MIN_VERIFIED_SAMPLES; + const maxSuccessRateRegression = + input.maxSuccessRateRegression ?? + DEFAULT_ORACLE_ROLLOUT_MAX_SUCCESS_RATE_REGRESSION; + const controlSuccessRate = rate( + input.control.successes, + input.control.verifiedSamples, + ); + const treatmentSuccessRate = rate( + input.treatment.successes, + input.treatment.verifiedSamples, + ); + const metrics = Object.freeze({ + controlSuccessRate, + treatmentSuccessRate, + successRateDelta: treatmentSuccessRate - controlSuccessRate, + costRatio: ratio( + input.treatment.averageCostUsd, + input.control.averageCostUsd, + ), + latencyRatio: ratio( + input.treatment.averageLatencyMs, + input.control.averageLatencyMs, + ), + }); + + const finish = ( + status: OraclePolicyRolloutStatus, + sufficient: boolean, + reasons: string[], + ): OraclePolicyRolloutDecision => + Object.freeze({ + experimentId: input.experimentId, + status, + sufficient, + reasons: Object.freeze(reasons), + metrics, + }); + const hasInvalidThreshold = + !Number.isInteger(minVerifiedSamples) || + minVerifiedSamples < 1 || + !Number.isFinite(maxSuccessRateRegression) || + maxSuccessRateRegression < 0 || + maxSuccessRateRegression > 1 || + (input.maxCostRatio !== undefined && + (!Number.isFinite(input.maxCostRatio) || input.maxCostRatio <= 0)) || + (input.maxLatencyRatio !== undefined && + (!Number.isFinite(input.maxLatencyRatio) || input.maxLatencyRatio <= 0)); + if (hasInvalidThreshold) { + return finish("hold", false, ["invalid_thresholds"]); + } + + if ( + input.control.policyVersion !== input.expectedControlVersion || + input.treatment.policyVersion !== input.expectedTreatmentVersion + ) { + return finish("hold", false, ["mixed_policy_versions"]); + } + + const safetyReasons: string[] = []; + if (input.control.safetyViolations > 0) { + safetyReasons.push("control_safety_violation"); + } + if (input.treatment.safetyViolations > 0) { + safetyReasons.push("treatment_safety_violation"); + } + if (safetyReasons.length > 0) return finish("rollback", true, safetyReasons); + + if ( + input.control.verifiedSamples < minVerifiedSamples || + input.treatment.verifiedSamples < minVerifiedSamples + ) { + return finish("hold", false, ["insufficient_verified_samples"]); + } + + const gateReasons: string[] = []; + if (metrics.successRateDelta < -maxSuccessRateRegression) { + gateReasons.push("success_rate_regression"); + } + if ( + input.maxCostRatio !== undefined && + metrics.costRatio > input.maxCostRatio + ) { + gateReasons.push("cost_ratio_exceeded"); + } + if ( + input.maxLatencyRatio !== undefined && + metrics.latencyRatio > input.maxLatencyRatio + ) { + gateReasons.push("latency_ratio_exceeded"); + } + + return gateReasons.length > 0 + ? finish("hold", true, gateReasons) + : finish("promote", true, ["all_rollout_gates_passed"]); +} diff --git a/src/agent/plugin-agent-api.ts b/src/agent/plugin-agent-api.ts new file mode 100644 index 000000000..46e287f4b --- /dev/null +++ b/src/agent/plugin-agent-api.ts @@ -0,0 +1,40 @@ +import { + type PluginAgentConfig, + type PluginAgentHandle, + type PluginAgentModeMetadata, + type PluginAgentModeRegistration, + type PluginAgentPolicy, + PluginAgentRegistry, + type RegisteredPluginAgentMode, +} from "./plugin-agent-registry.js"; + +export interface PluginAgentApi { + createAgent(config: PluginAgentConfig): PluginAgentHandle; + registerAgentMode(registration: PluginAgentModeRegistration): void; + getAgentMode(key: string): RegisteredPluginAgentMode | undefined; + listAgentModes(): readonly RegisteredPluginAgentMode[]; +} + +export function createPluginAgentApi(input: { + policy: PluginAgentPolicy; + metadata: readonly PluginAgentModeMetadata[]; +}): PluginAgentApi { + const registry = new PluginAgentRegistry(input.policy, input.metadata); + return Object.freeze({ + createAgent: registry.createAgent.bind(registry), + registerAgentMode: registry.registerAgentMode.bind(registry), + getAgentMode: registry.getAgentMode.bind(registry), + listAgentModes: registry.listAgentModes.bind(registry), + }); +} + +export type { + PluginAgentBudgets, + PluginAgentConfig, + PluginAgentHandle, + PluginAgentModeMetadata, + PluginAgentModeRegistration, + PluginAgentPolicy, + PluginAgentSandboxMode, + RegisteredPluginAgentMode, +} from "./plugin-agent-registry.js"; diff --git a/src/agent/plugin-agent-loader.ts b/src/agent/plugin-agent-loader.ts new file mode 100644 index 000000000..ba742a65c --- /dev/null +++ b/src/agent/plugin-agent-loader.ts @@ -0,0 +1,112 @@ +import { existsSync, readFileSync, realpathSync, statSync } from "node:fs"; +import { basename, isAbsolute, relative, resolve } from "node:path"; +import type { ConfiguredPackageRuntimeOptions } from "../packages/runtime.js"; +import { loadConfiguredPackageResources } from "../packages/runtime.js"; +import type { PluginAgentModeMetadata } from "./plugin-agent-registry.js"; + +const AGENT_METADATA_FILE = "agent.json"; +const MAX_AGENT_METADATA_BYTES = 64 * 1024; +const KEY_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; + +export interface LoadedPluginAgentModeMetadata extends PluginAgentModeMetadata { + readonly directory: string; + readonly scope: "user" | "project"; +} + +export interface PluginAgentMetadataLoadResult { + metadata: readonly LoadedPluginAgentModeMetadata[]; + errors: readonly string[]; +} + +function isContainedPath(parent: string, child: string): boolean { + const path = relative(parent, child); + return path === "" || (!path.startsWith("..") && !isAbsolute(path)); +} + +function loadMetadataDirectory( + directory: string, + scope: "user" | "project", +): LoadedPluginAgentModeMetadata { + const metadataPath = resolve(directory, AGENT_METADATA_FILE); + if (!existsSync(metadataPath) || !statSync(metadataPath).isFile()) { + throw new Error(`Missing ${AGENT_METADATA_FILE}`); + } + if (statSync(metadataPath).size > MAX_AGENT_METADATA_BYTES) { + throw new Error(`${AGENT_METADATA_FILE} exceeds 64 KiB`); + } + const value = JSON.parse(readFileSync(metadataPath, "utf8")) as { + key?: unknown; + label?: unknown; + entry?: unknown; + }; + if (typeof value.key !== "string" || !KEY_PATTERN.test(value.key)) { + throw new Error("Agent key must be lowercase kebab-case"); + } + if (value.key !== basename(directory)) { + throw new Error("Agent key must match its directory name"); + } + if (typeof value.label !== "string" || !value.label.trim()) { + throw new Error("Agent label must be non-empty"); + } + if (typeof value.entry !== "string" || !value.entry.trim()) { + throw new Error("Agent entry must be non-empty"); + } + const realDirectory = realpathSync(directory); + const entryPath = resolve(realDirectory, value.entry); + if (!existsSync(entryPath) || !statSync(entryPath).isFile()) { + throw new Error(`Agent entry does not exist: ${value.entry}`); + } + const realEntry = realpathSync(entryPath); + if (!isContainedPath(realDirectory, realEntry)) { + throw new Error("Agent entry must remain inside its resource directory"); + } + return Object.freeze({ + key: value.key, + label: value.label.trim(), + entry: realEntry, + directory: realDirectory, + scope, + }); +} + +export function loadPluginAgentMetadataFromDirectories(input: { + user?: readonly string[]; + project?: readonly string[]; + initialErrors?: readonly string[]; +}): PluginAgentMetadataLoadResult { + const metadata: LoadedPluginAgentModeMetadata[] = []; + const errors = [...(input.initialErrors ?? [])]; + const seen = new Set(); + for (const scope of ["user", "project"] as const) { + for (const directory of input[scope] ?? []) { + try { + const loaded = loadMetadataDirectory(directory, scope); + if (seen.has(loaded.key)) { + throw new Error(`Duplicate plugin agent key: ${loaded.key}`); + } + seen.add(loaded.key); + metadata.push(loaded); + } catch (error) { + errors.push( + `${directory}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + } + return Object.freeze({ + metadata: Object.freeze(metadata), + errors: Object.freeze(errors), + }); +} + +export function loadConfiguredPluginAgentMetadata( + workspaceDir: string, + options: ConfiguredPackageRuntimeOptions = {}, +): PluginAgentMetadataLoadResult { + const resources = loadConfiguredPackageResources(workspaceDir, options); + return loadPluginAgentMetadataFromDirectories({ + user: resources.agents.user, + project: resources.agents.project, + initialErrors: resources.errors, + }); +} diff --git a/src/agent/plugin-agent-registry.ts b/src/agent/plugin-agent-registry.ts new file mode 100644 index 000000000..4dc3d1666 --- /dev/null +++ b/src/agent/plugin-agent-registry.ts @@ -0,0 +1,225 @@ +import type { ApprovalMode } from "./action-approval.js"; + +export type PluginAgentSandboxMode = + | "danger-full-access" + | "workspace-write" + | "read-only"; + +export interface PluginAgentBudgets { + maxTurns: number; + maxToolCalls: number; + maxCostUsd: number; +} + +export interface PluginAgentPolicy { + allowedModels: readonly string[]; + allowedTools: readonly string[]; + maxBudgets: PluginAgentBudgets; + approvalMode: ApprovalMode; + sandboxMode: PluginAgentSandboxMode; +} + +export interface PluginAgentModeMetadata { + key: string; + label: string; + entry: string; +} + +export interface PluginAgentConfig { + key: string; + label: string; + description: string; + systemPrompt: string; + model: string; + tools: "all" | readonly string[]; + budgets: PluginAgentBudgets; + approvalMode: ApprovalMode; + sandboxMode: PluginAgentSandboxMode; +} + +export interface PluginAgentHandle { + readonly key: string; + readonly label: string; + readonly description: string; + readonly systemPrompt: string; + readonly model: string; + readonly tools: readonly string[]; + readonly budgets: Readonly; + readonly approvalMode: ApprovalMode; + readonly sandboxMode: PluginAgentSandboxMode; +} + +export interface PluginAgentModeRegistration { + key: string; + label: string; + agent: PluginAgentHandle; + primary: boolean; +} + +export interface RegisteredPluginAgentMode extends PluginAgentModeRegistration { + readonly metadata: Readonly; +} + +const KEY_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; +const APPROVAL_RESTRICTIVENESS: Record = { + auto: 0, + prompt: 1, + fail: 2, +}; +const SANDBOX_RESTRICTIVENESS: Record = { + "danger-full-access": 0, + "workspace-write": 1, + "read-only": 2, +}; + +function validatePositiveBudget( + name: keyof PluginAgentBudgets, + requested: number | undefined, + maximum: number | undefined, +): void { + if ( + requested === undefined || + !Number.isFinite(requested) || + requested <= 0 || + (maximum !== undefined && requested > maximum) + ) { + throw new Error(`Plugin agent budget ${name} is invalid or exceeds policy`); + } +} + +function freezeHandle( + config: PluginAgentConfig, + tools: readonly string[], +): PluginAgentHandle { + return Object.freeze({ + key: config.key, + label: config.label, + description: config.description, + systemPrompt: config.systemPrompt, + model: config.model, + tools: Object.freeze([...tools]), + budgets: Object.freeze({ ...config.budgets }), + approvalMode: config.approvalMode, + sandboxMode: config.sandboxMode, + }); +} + +export class PluginAgentRegistry { + private readonly policy: PluginAgentPolicy; + private readonly metadata = new Map< + string, + Readonly + >(); + private readonly registrations = new Map(); + private readonly issuedHandles = new WeakSet(); + + constructor( + policy: PluginAgentPolicy, + metadata: readonly PluginAgentModeMetadata[], + ) { + this.policy = Object.freeze({ + ...policy, + allowedModels: Object.freeze([...policy.allowedModels]), + allowedTools: Object.freeze([...policy.allowedTools]), + maxBudgets: Object.freeze({ ...policy.maxBudgets }), + }); + for (const declaration of metadata) { + if (!KEY_PATTERN.test(declaration.key) || !declaration.entry.trim()) { + throw new Error(`Invalid plugin agent metadata for ${declaration.key}`); + } + if (this.metadata.has(declaration.key)) { + throw new Error(`Duplicate plugin agent metadata: ${declaration.key}`); + } + this.metadata.set(declaration.key, Object.freeze({ ...declaration })); + } + } + + createAgent(config: PluginAgentConfig): PluginAgentHandle { + if (!KEY_PATTERN.test(config.key)) { + throw new Error("Plugin agent key must be lowercase kebab-case"); + } + if ( + !config.label.trim() || + !config.description.trim() || + !config.systemPrompt.trim() + ) { + throw new Error("Plugin agent text fields must be non-empty"); + } + if (!this.policy.allowedModels.includes(config.model)) { + throw new Error(`Plugin agent model is not allowed: ${config.model}`); + } + + const tools = + config.tools === "all" ? this.policy.allowedTools : config.tools; + const unknownTool = tools.find( + (tool) => !this.policy.allowedTools.includes(tool), + ); + if (unknownTool) { + throw new Error(`Plugin agent tool is not allowed: ${unknownTool}`); + } + validatePositiveBudget( + "maxTurns", + config.budgets.maxTurns, + this.policy.maxBudgets.maxTurns, + ); + validatePositiveBudget( + "maxToolCalls", + config.budgets.maxToolCalls, + this.policy.maxBudgets.maxToolCalls, + ); + validatePositiveBudget( + "maxCostUsd", + config.budgets.maxCostUsd, + this.policy.maxBudgets.maxCostUsd, + ); + if ( + APPROVAL_RESTRICTIVENESS[config.approvalMode] < + APPROVAL_RESTRICTIVENESS[this.policy.approvalMode] + ) { + throw new Error("Plugin agent approval mode would escalate permission"); + } + if ( + SANDBOX_RESTRICTIVENESS[config.sandboxMode] < + SANDBOX_RESTRICTIVENESS[this.policy.sandboxMode] + ) { + throw new Error("Plugin agent sandbox mode would escalate permission"); + } + + const handle = freezeHandle(config, tools); + this.issuedHandles.add(handle); + return handle; + } + + registerAgentMode(registration: PluginAgentModeRegistration): void { + if (this.registrations.has(registration.key)) { + throw new Error(`Duplicate plugin agent mode: ${registration.key}`); + } + const metadata = this.metadata.get(registration.key); + if ( + !metadata || + metadata.label !== registration.label || + registration.agent.key !== registration.key || + registration.agent.label !== registration.label + ) { + throw new Error( + `Plugin agent registration metadata mismatch: ${registration.key}`, + ); + } + if (!this.issuedHandles.has(registration.agent)) { + throw new Error("Plugin agent handle was not issued by this registry"); + } + + this.registrations.set( + registration.key, + Object.freeze({ ...registration, metadata }), + ); + } + + getAgentMode(key: string): RegisteredPluginAgentMode | undefined { + return this.registrations.get(key); + } + + listAgentModes(): readonly RegisteredPluginAgentMode[] { + return Object.freeze([...this.registrations.values()]); + } +} diff --git a/src/agent/routing-receipt.ts b/src/agent/routing-receipt.ts new file mode 100644 index 000000000..6ab476c21 --- /dev/null +++ b/src/agent/routing-receipt.ts @@ -0,0 +1,95 @@ +import type { + AgentProfilePin, + RoutingReceipt, + RoutingReceiptExperiment, + RoutingReceiptSource, +} from "@evalops/contracts"; +import type { OracleConsultationDecision } from "./oracle-consultation-policy.js"; +import type { AgentProfile } from "./profiles.js"; + +export interface AgentProfileSelectionInput { + requestedProfile?: string; + sessionPin?: AgentProfilePin; + compatibilityProfile: string; +} + +export interface AgentProfileSelection { + requestedProfile: string; + source: RoutingReceiptSource; +} + +export interface RoutingReceiptDecision { + decisionId: string; + selectedModel: { provider: string; model: string }; + selectedProfile: AgentProfile; + createdAt: string; + oracleConsultation?: OracleConsultationDecision; +} + +export interface RoutingReceiptContext extends AgentProfileSelection { + fallbackReason?: string; + fallbackModel?: { provider: string; model: string }; + experiment?: RoutingReceiptExperiment; +} + +export function resolveAgentProfileSelection( + input: AgentProfileSelectionInput, +): AgentProfileSelection { + if (input.requestedProfile?.trim()) { + return { + requestedProfile: input.requestedProfile.trim(), + source: "request", + }; + } + if (input.sessionPin?.profile.trim()) { + return { + requestedProfile: input.sessionPin.profile.trim(), + source: "session", + }; + } + if (!input.compatibilityProfile.trim()) { + throw new Error("A compatibility profile is required"); + } + return { + requestedProfile: input.compatibilityProfile.trim(), + source: "compatibility_default", + }; +} + +/** Project a routing decision into immutable historical turn metadata. */ +export function createRoutingReceipt( + decision: RoutingReceiptDecision, + context: RoutingReceiptContext, +): RoutingReceipt { + const oracle = decision.oracleConsultation + ? Object.freeze({ + policyVersion: decision.oracleConsultation.policyVersion, + mode: decision.oracleConsultation.mode, + reasons: Object.freeze([...decision.oracleConsultation.reasons]), + }) + : undefined; + const fallback = context.fallbackReason + ? Object.freeze({ + reason: context.fallbackReason, + ...(context.fallbackModel ?? {}), + }) + : undefined; + const experiment = context.experiment + ? Object.freeze({ ...context.experiment }) + : undefined; + + return Object.freeze({ + decisionId: decision.decisionId, + requestedProfile: context.requestedProfile, + source: context.source, + resolvedProfileId: decision.selectedProfile.id, + resolvedProfileVersion: decision.selectedProfile.version, + provider: decision.selectedModel.provider, + model: decision.selectedModel.model, + reasoningEffort: decision.selectedProfile.primary.reasoningEffort, + createdAt: decision.createdAt, + ...(oracle ? { oracle } : {}), + ...(fallback ? { fallback } : {}), + ...(experiment ? { experiment } : {}), + }); +} diff --git a/src/agent/subagent-specs.ts b/src/agent/subagent-specs.ts index c618cf234..57d0bc7d9 100644 --- a/src/agent/subagent-specs.ts +++ b/src/agent/subagent-specs.ts @@ -106,7 +106,7 @@ export const TOOL_CATEGORIES = { /** GitHub tools */ github: ["gh_pr", "gh_issue", "gh_repo"], /** Advanced tools */ - advanced: ["oracle"], + advanced: ["oracle", "painter"], } as const; /** @@ -152,7 +152,7 @@ export const SUBAGENT_SPECS: Record = { displayName: "Explorer", description: "Read-only codebase exploration - can search and read files", allowedTools: toolsFromCategories("read", "advanced"), - deniedTools: ["oracle"], // Explorers shouldn't invoke oracle + deniedTools: ["oracle", "painter"], // Explorers are read-only; no reasoning subagents or image generation allowMcp: false, allowToolbox: false, maxToolCallsPerTurn: 20, diff --git a/src/agent/types.ts b/src/agent/types.ts index 200b0d058..3861e5d63 100644 --- a/src/agent/types.ts +++ b/src/agent/types.ts @@ -31,7 +31,10 @@ * @module agent/types */ -import type { GuardedFilesPolicySettings } from "@evalops/contracts"; +import type { + GuardedFilesPolicySettings, + RoutingReceipt, +} from "@evalops/contracts"; import type { TSchema } from "@sinclair/typebox"; import type { PromptProjectDocManifest } from "../config/index.js"; import type { UnifiedContextManifest } from "../context/manifest-types.js"; @@ -452,6 +455,8 @@ export interface AssistantMessage { errorMessage?: string; /** Unix timestamp in milliseconds when message was created */ timestamp: number; + /** Immutable model/profile routing decision for this turn. */ + routingReceipt?: RoutingReceipt; } /** diff --git a/src/app-server/plugin-bundle-api.ts b/src/app-server/plugin-bundle-api.ts index a055a01f9..0b81b261d 100644 --- a/src/app-server/plugin-bundle-api.ts +++ b/src/app-server/plugin-bundle-api.ts @@ -236,6 +236,7 @@ export function createMaestroAppServerPluginBundleApi( configPath: entry.configPath, })), resources: { + agents: resources.agents, extensions: resources.extensions, skills: resources.skills, prompts: resources.prompts, diff --git a/src/cli-tui/commands/package-handlers.ts b/src/cli-tui/commands/package-handlers.ts index 8e72f14e8..a3613e45b 100644 --- a/src/cli-tui/commands/package-handlers.ts +++ b/src/cli-tui/commands/package-handlers.ts @@ -425,6 +425,7 @@ function formatInspectReport(inspected: InspectedPackage, cwd: string): string { const manifest = packageJson.maestro; if (manifest) { lines.push(" Manifest paths:"); + lines.push(` Agents: ${formatManifestPaths(manifest.agents)}`); lines.push(` Extensions: ${formatManifestPaths(manifest.extensions)}`); lines.push(` Skills: ${formatManifestPaths(manifest.skills)}`); lines.push(` Prompts: ${formatManifestPaths(manifest.prompts)}`); @@ -441,6 +442,9 @@ function formatInspectReport(inspected: InspectedPackage, cwd: string): string { if (inspected.resources) { lines.push(" Resources:"); + lines.push( + ` Agents: ${formatResourceSummary(inspected.resources.agents)}`, + ); lines.push( ` Extensions: ${formatResourceSummary(inspected.resources.extensions)}`, ); @@ -494,7 +498,7 @@ function formatConfiguredPackageList( lines.push(` Name: ${inspected.discovered.packageJson.name}`); lines.push( - ` Resources: extensions=${inspected.resources?.extensions.length ?? 0}, skills=${inspected.resources?.skills.length ?? 0}, prompts=${inspected.resources?.prompts.length ?? 0}, themes=${inspected.resources?.themes.length ?? 0}`, + ` Resources: agents=${inspected.resources?.agents.length ?? 0}, extensions=${inspected.resources?.extensions.length ?? 0}, skills=${inspected.resources?.skills.length ?? 0}, prompts=${inspected.resources?.prompts.length ?? 0}, themes=${inspected.resources?.themes.length ?? 0}`, ); } @@ -513,6 +517,7 @@ function formatValidationSuccess( lines.push(` Path: ${formatDisplayPath(inspected.resolvedPath, cwd)}`); if (inspected.resources) { lines.push(" Resources:"); + lines.push(` Agents: ${inspected.resources.agents.length}`); lines.push(` Extensions: ${inspected.resources.extensions.length}`); lines.push(` Skills: ${inspected.resources.skills.length}`); lines.push(` Prompts: ${inspected.resources.prompts.length}`); @@ -614,7 +619,13 @@ function formatFilters(filters: ResourceFilters | undefined): string | null { } const segments: string[] = []; - for (const key of ["extensions", "skills", "prompts", "themes"] as const) { + for (const key of [ + "agents", + "extensions", + "skills", + "prompts", + "themes", + ] as const) { const values = filters[key]; if (values && values.length > 0) { segments.push(`${key}=${values.join(",")}`); diff --git a/src/cli/args.ts b/src/cli/args.ts index 73bdab533..319f825b3 100644 --- a/src/cli/args.ts +++ b/src/cli/args.ts @@ -91,6 +91,7 @@ const COMMANDS = new Set([ "hooks", "skill", "memory", + "painter", "remote", "export", "import", @@ -115,6 +116,7 @@ const SUBCOMMAND_COMMANDS = new Set([ "hooks", "skill", "memory", + "painter", "remote", "scenario", ]); @@ -433,7 +435,8 @@ export function parseArgs(args: string[]): Args { arg === "init" || arg === "evalops" || arg === "update" || - arg === "skill" + arg === "skill" || + arg === "painter" ) { result.commandArgs = args.slice(i + 1); const commandTailStreamJson = commandTailStreamJsonFlag( diff --git a/src/cli/commands/painter.ts b/src/cli/commands/painter.ts new file mode 100644 index 000000000..8de3eef27 --- /dev/null +++ b/src/cli/commands/painter.ts @@ -0,0 +1,104 @@ +/** + * `maestro painter` CLI command. + * + * Today this exposes `maestro painter show `, which renders an image + * inline in a capable terminal (iTerm2/WezTerm/kitty) by writing the protocol + * escape straight to stdout. This is the display-only integration for the + * terminal-image emitter: it never routes through the agent loop, so it cannot + * waste model tokens or corrupt the conversation context. + * + * Run from a plain shell, not inside the full-screen TUI (the TUI owns the + * screen and would overwrite raw writes). + */ + +import { readFile } from "node:fs/promises"; +import { basename, resolve } from "node:path"; +import { + type TerminalImageSupport, + detectTerminalImageSupport, + encodeInlineImage, +} from "../../utils/terminal-image.js"; + +export function formatPainterHelp(): string { + return [ + "Usage:", + " maestro painter show Render an image inline in a capable terminal", + "", + "Requires iTerm2, WezTerm, or kitty. Run from a plain shell, not inside", + "the full-screen TUI.", + ].join("\n"); +} + +export interface InlineDisplayResult { + ok: boolean; + escape?: string; + reason?: string; +} + +/** + * Pure policy: decide whether an image buffer can be shown inline and produce + * the escape sequence. Separated from IO so it is unit-testable. + */ +export function resolveInlineDisplay( + buf: Buffer, + opts: { name?: string; isTTY?: boolean; support?: TerminalImageSupport } = {}, +): InlineDisplayResult { + if (opts.isTTY === false) { + return { + ok: false, + reason: "stdout is not a TTY; inline image display requires a terminal.", + }; + } + const support = opts.support ?? detectTerminalImageSupport(); + const sequence = encodeInlineImage(buf, support, { name: opts.name }); + if (!sequence) { + return { + ok: false, + reason: `Inline image display is not supported in this terminal (detected: ${support}). Use iTerm2, WezTerm, or kitty.`, + }; + } + return { ok: true, escape: sequence }; +} + +export async function handlePainterCommand( + subcommand: string | undefined, + args: string[], + out: NodeJS.WritableStream = process.stdout, +): Promise { + if (subcommand === "show") { + await showImage(args[0], out); + return; + } + out.write(`${formatPainterHelp()}\n`); +} + +async function showImage( + pathArg: string | undefined, + out: NodeJS.WritableStream, +): Promise { + if (!pathArg) { + console.error("maestro painter show requires an image path."); + process.exitCode = 1; + return; + } + const path = resolve(pathArg); + let buf: Buffer; + try { + buf = await readFile(path); + } catch { + console.error(`Could not read image: ${path}`); + process.exitCode = 1; + return; + } + + const result = resolveInlineDisplay(buf, { + name: basename(path), + isTTY: (out as { isTTY?: boolean }).isTTY, + }); + if (!result.ok || !result.escape) { + console.error(result.reason ?? "Inline image display failed."); + process.exitCode = 1; + return; + } + out.write(result.escape); +} diff --git a/src/cli/help.ts b/src/cli/help.ts index 1091c2cb1..53176bbdd 100644 --- a/src/cli/help.ts +++ b/src/cli/help.ts @@ -187,6 +187,9 @@ export function printHelp( maestro skill new processing-incidents --description "Process incident reports. Use when the user asks for incident triage." maestro skill lint .maestro/skills + # Render a generated image inline in a capable terminal (iTerm2/WezTerm/kitty) + maestro painter show ~/.maestro/assets/painter/painter-*.png + # Start a hosted EvalOps runner session and wait for attach readiness maestro remote start --workspace ws_123 --repo evalops/foo --branch main --ttl 90m --wait --verify diff --git a/src/config/env-vars.ts b/src/config/env-vars.ts index 9eb9de07a..0c3e4431d 100644 --- a/src/config/env-vars.ts +++ b/src/config/env-vars.ts @@ -27,6 +27,10 @@ * ### Subagent Configuration * - MAESTRO_SUBAGENT_MODEL - Model to use for subagents * - MAESTRO_ORACLE_MODEL - Model to use for the Oracle tool + * - MAESTRO_PAINTER_MODEL - Image model for the Painter tool (default: gpt-image-2) + * - MAESTRO_PAINTER_BASE_URL - OpenAI-compatible base URL override for Painter + * - MAESTRO_PAINTER_OUTPUT_DIR - Where Painter writes images (default: ~/.maestro/assets/painter) + * - MAESTRO_PAINTER_TIMEOUT_MS - Per-call Painter timeout in ms (default: 180000) * * ### API Configuration * - MAESTRO_API_KEY_HELPER_TTL_MS - Cache TTL for API key helpers @@ -88,6 +92,16 @@ export const ENV_VARS = { // Subagent configuration SUBAGENT_MODEL: "MAESTRO_SUBAGENT_MODEL", ORACLE_MODEL: "MAESTRO_ORACLE_MODEL", + ORACLE_EXPERIMENT_ID: "MAESTRO_ORACLE_EXPERIMENT_ID", + ORACLE_EXPERIMENT_ALLOCATION: "MAESTRO_ORACLE_EXPERIMENT_ALLOCATION", + ORACLE_EXPERIMENT_CONTROL_VERSION: + "MAESTRO_ORACLE_EXPERIMENT_CONTROL_VERSION", + ORACLE_EXPERIMENT_TREATMENT_VERSION: + "MAESTRO_ORACLE_EXPERIMENT_TREATMENT_VERSION", + PAINTER_MODEL: "MAESTRO_PAINTER_MODEL", + PAINTER_BASE_URL: "MAESTRO_PAINTER_BASE_URL", + PAINTER_OUTPUT_DIR: "MAESTRO_PAINTER_OUTPUT_DIR", + PAINTER_TIMEOUT_MS: "MAESTRO_PAINTER_TIMEOUT_MS", SWARM_MODE: "MAESTRO_SWARM_MODE", SWARM_ID: "MAESTRO_SWARM_ID", TEAMMATE_ID: "MAESTRO_TEAMMATE_ID", diff --git a/src/main.ts b/src/main.ts index fff4a0b31..694636884 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1283,6 +1283,11 @@ export async function main(args: string[]) { parsed.command === "exec" ? import("./cli/commands/exec.js") : undefined; // Handle cost commands + if (parsed.command === "painter") { + const { handlePainterCommand } = await import("./cli/commands/painter.js"); + await handlePainterCommand(parsed.subcommand, parsed.commandArgs ?? []); + return; + } if (parsed.command === "cost") { const { handleCostSummary, handleCostClear, handleCostBreakdown } = await import("./cli/commands/cost.js"); diff --git a/src/packages/discovery.ts b/src/packages/discovery.ts index f05cd4abc..6a9075513 100644 --- a/src/packages/discovery.ts +++ b/src/packages/discovery.ts @@ -50,6 +50,7 @@ export function discoverPackage(packagePath: string): DiscoveredPackage | null { // Validate that resource paths are arrays for (const key of [ + "agents", "extensions", "skills", "prompts", diff --git a/src/packages/inspection.ts b/src/packages/inspection.ts index bd89f2c3d..50bcbb808 100644 --- a/src/packages/inspection.ts +++ b/src/packages/inspection.ts @@ -99,7 +99,13 @@ function collectManifestPathIssues( manifest: MaestroManifest, ): string[] { const issues: string[] = []; - for (const key of ["extensions", "skills", "prompts", "themes"] as const) { + for (const key of [ + "agents", + "extensions", + "skills", + "prompts", + "themes", + ] as const) { const manifestPaths = manifest[key]; if (!Array.isArray(manifestPaths)) { continue; diff --git a/src/packages/loader.ts b/src/packages/loader.ts index ab6237d93..dc7ca0052 100644 --- a/src/packages/loader.ts +++ b/src/packages/loader.ts @@ -40,9 +40,10 @@ export function parsePackageSpec( } // Object form with filters - const { source, extensions, skills, prompts, themes } = spec; + const { source, agents, extensions, skills, prompts, themes } = spec; const filters: ResourceFilters = {}; + if (agents) filters.agents = agents; if (extensions) filters.extensions = extensions; if (skills) filters.skills = skills; if (prompts) filters.prompts = prompts; @@ -118,12 +119,21 @@ export async function loadPackage( export function loadPackageResources(pkg: LoadedPackage): PackageResources { const resources: PackageResources = { package: pkg, + agents: [], extensions: [], skills: [], prompts: [], themes: [], }; + if (pkg.manifest.agents) { + resources.agents = loadResourcePaths( + pkg.path, + pkg.manifest.agents, + pkg.filters?.agents, + ); + } + // Load each resource type if (pkg.manifest.extensions) { resources.extensions = loadResourcePaths( @@ -159,6 +169,7 @@ export function loadPackageResources(pkg: LoadedPackage): PackageResources { logger.debug("Loaded package resources", { package: pkg.name, + agents: resources.agents.length, extensions: resources.extensions.length, skills: resources.skills.length, prompts: resources.prompts.length, diff --git a/src/packages/runtime.ts b/src/packages/runtime.ts index fbe2a094d..421813448 100644 --- a/src/packages/runtime.ts +++ b/src/packages/runtime.ts @@ -16,7 +16,13 @@ import type { LoadedPackage, PackageResources } from "./types.js"; const logger = createLogger("packages:runtime"); -const RESOURCE_KINDS = ["extensions", "skills", "prompts", "themes"] as const; +const RESOURCE_KINDS = [ + "agents", + "extensions", + "skills", + "prompts", + "themes", +] as const; type ResourceKind = (typeof RESOURCE_KINDS)[number]; type PackageScope = ConfiguredPackageSpec["scope"]; @@ -28,6 +34,7 @@ export interface ScopedPackageResourceDirectories { } export interface ConfiguredPackageRuntimeResources { + agents: ScopedPackageResourceDirectories; extensions: ScopedPackageResourceDirectories; skills: ScopedPackageResourceDirectories; prompts: ScopedPackageResourceDirectories; @@ -59,6 +66,7 @@ function createScopedDirectories(): ScopedPackageResourceDirectories { function createConfiguredPackageRuntimeResources(): ConfiguredPackageRuntimeResources { return { + agents: createScopedDirectories(), extensions: createScopedDirectories(), skills: createScopedDirectories(), prompts: createScopedDirectories(), @@ -169,6 +177,7 @@ export function loadConfiguredPackageResources( }); const resources = createConfiguredPackageRuntimeResources(); const seen: Record>> = { + agents: { user: new Set(), project: new Set() }, extensions: { user: new Set(), project: new Set() }, skills: { user: new Set(), project: new Set() }, prompts: { user: new Set(), project: new Set() }, diff --git a/src/packages/types.ts b/src/packages/types.ts index e0ea2bd5b..9bec88553 100644 --- a/src/packages/types.ts +++ b/src/packages/types.ts @@ -9,6 +9,8 @@ * Package manifest (maestro section in package.json) */ export interface MaestroManifest { + /** Paths to custom agent directories */ + agents?: string[]; /** Paths to extension directories */ extensions?: string[]; /** Paths to skill directories */ @@ -69,6 +71,8 @@ export type PackageSource = GitSource | LocalSource | NpmSource; * Resource filter patterns */ export interface ResourceFilters { + /** Glob patterns for custom agents */ + agents?: string[]; /** Glob patterns for extensions (supports ! for exclusion) */ extensions?: string[]; /** Glob patterns for skills */ @@ -110,6 +114,8 @@ export interface LoadedPackage { export interface PackageResources { /** Package metadata */ package: LoadedPackage; + /** Loaded custom agent paths */ + agents: string[]; /** Loaded extension paths */ extensions: string[]; /** Loaded skill paths */ diff --git a/src/server/handlers/chat-ws.ts b/src/server/handlers/chat-ws.ts index 42392f7d6..f69e67af2 100644 --- a/src/server/handlers/chat-ws.ts +++ b/src/server/handlers/chat-ws.ts @@ -6,9 +6,18 @@ import { randomBytes, randomUUID } from "node:crypto"; import type { IncomingMessage } from "node:http"; -import type { ComposerChatRequest, ComposerMessage } from "@evalops/contracts"; +import type { + ComposerChatRequest, + ComposerMessage, + RoutingReceipt, +} from "@evalops/contracts"; import type { RawData, WebSocket } from "ws"; import { applyOracleConsultationDirective } from "../../agent/oracle-consultation-policy.js"; +import { assignConfiguredOraclePolicyExperiment } from "../../agent/oracle-policy-experiment.js"; +import { + createRoutingReceipt, + resolveAgentProfileSelection, +} from "../../agent/routing-receipt.js"; import { isAssistantMessage } from "../../agent/type-guards.js"; import type { Attachment as AgentAttachment, @@ -26,6 +35,7 @@ import { createAutomaticMemoryExtractionCoordinator } from "../../memory/auto-ex import type { RegisteredModel } from "../../models/registry.js"; import { recordIntelligentRouterChatMetric, + resolveIntelligentRouterProfileHint, selectIntelligentRouterModel, } from "../../services/intelligent-router/recorder.js"; import { recordAssistantUsageMetric } from "../../services/usage-analytics/recorder.js"; @@ -33,6 +43,7 @@ import { evaluateModelPolicy } from "../../services/workspace-config/policy.js"; import type { WorkspaceConfigRequestContext } from "../../services/workspace-config/types.js"; import { toSessionModelMetadata } from "../../session/manager.js"; import { createRuntimeSessionSummaryUpdater } from "../../session/runtime-summary-updater.js"; +import { recordOraclePolicyExperimentAssignment } from "../../telemetry/oracle-policy.js"; import { createLogger } from "../../utils/logger.js"; import type { WebServerContext } from "../app-context.js"; import { @@ -136,6 +147,10 @@ class WsSession { this.write(JSON.stringify({ type: "session_update", sessionId })); } + sendRoutingReceipt(receipt: RoutingReceipt): void { + this.write(JSON.stringify({ type: "routing_receipt", receipt })); + } + sendHeartbeat(): void { this.write(JSON.stringify({ type: "heartbeat" })); } @@ -479,6 +494,17 @@ export function handleChatWebSocket( } } + const profileSelection = resolveAgentProfileSelection({ + requestedProfile: resolveIntelligentRouterProfileHint(req, chatReq), + sessionPin: sessionManager.getHeader()?.agentProfilePin, + compatibilityProfile: "medium", + }); + if (chatReq.persistProfile && profileSelection.source === "request") { + sessionManager.updateAgentProfilePin({ + profile: profileSelection.requestedProfile, + updatedAt: new Date().toISOString(), + }); + } const routingSelection = selectIntelligentRouterModel({ req, requestedModel: resolveModelInputForRouting( @@ -486,9 +512,30 @@ export function handleChatWebSocket( defaultProvider, defaultModelId, ), - body: chatReq, + body: { ...chatReq, profile: profileSelection.requestedProfile }, }); + const oracleExperimentAssignment = assignConfiguredOraclePolicyExperiment( + sessionManager.getSessionId(), + ); + if (oracleExperimentAssignment) { + recordOraclePolicyExperimentAssignment({ + assignment: oracleExperimentAssignment, + sessionId: sessionManager.getSessionId(), + }); + } + const routedDecision = oracleExperimentAssignment + ? { + ...routingSelection.decision, + oracleConsultation: routingSelection.decision.oracleConsultation + ? { + ...routingSelection.decision.oracleConsultation, + policyVersion: oracleExperimentAssignment.policyVersion, + } + : undefined, + } + : routingSelection.decision; let registeredModel: RegisteredModel | undefined; + let selectedModelInputIndex = -1; let lastModelError: unknown; let selectedModelError: unknown; let selectedModelPolicyViolation: ReturnType< @@ -512,6 +559,7 @@ export function handleChatWebSocket( continue; } registeredModel = candidateModel; + selectedModelInputIndex = index; break; } catch (error) { lastModelError = error; @@ -527,6 +575,42 @@ export function handleChatWebSocket( return; } if (!registeredModel) throw selectedModelError ?? lastModelError; + const usedFallback = selectedModelInputIndex > 0; + const routingReceipt = createRoutingReceipt( + { + ...routedDecision, + selectedModel: { + provider: registeredModel.provider, + model: registeredModel.id, + }, + }, + { + ...profileSelection, + ...(oracleExperimentAssignment + ? { + experiment: { + experimentId: oracleExperimentAssignment.experimentId, + arm: oracleExperimentAssignment.arm, + policyVersion: oracleExperimentAssignment.policyVersion, + }, + } + : {}), + ...(usedFallback || + routingSelection.decision.reason !== "highest_score" + ? { + fallbackReason: usedFallback + ? "selected_model_unavailable_or_disallowed" + : routingSelection.decision.reason, + fallbackModel: usedFallback + ? { + provider: registeredModel.provider, + model: registeredModel.id, + } + : undefined, + } + : {}), + }, + ); const routeStartedAt = Date.now(); const headerApproval = (() => { @@ -608,7 +692,7 @@ export function handleChatWebSocket( ); applyOracleConsultationDirective( agent, - routingSelection.decision.oracleConsultation, + routedDecision.oracleConsultation, ); const historyMessages = incomingMessages.slice(0, -1); @@ -625,6 +709,7 @@ export function handleChatWebSocket( const requestId = Math.random().toString(36).slice(2); const modelKey = `${registeredModel.provider}/${registeredModel.id}`; const wsSession = new WsSession(ws, undefined, { requestId, modelKey }); + wsSession.sendRoutingReceipt(routingReceipt); const { enterpriseContext } = await import("../../enterprise/context.js"); const initializeSessionIfNeeded = async (): Promise => { @@ -917,7 +1002,10 @@ export function handleChatWebSocket( } if (event.type === "message_end") { - sessionManager.saveMessage(event.message); + const persistedMessage = isAssistantMessage(event.message) + ? { ...event.message, routingReceipt } + : event.message; + sessionManager.saveMessage(persistedMessage); if (isAssistantMessage(event.message)) { recordAssistantUsageMetric({ req, diff --git a/src/server/handlers/chat.ts b/src/server/handlers/chat.ts index 7c455bbc9..c39d91ccc 100644 --- a/src/server/handlers/chat.ts +++ b/src/server/handlers/chat.ts @@ -24,6 +24,11 @@ import type { IncomingMessage, ServerResponse } from "node:http"; import type { ComposerChatRequest, ComposerMessage } from "@evalops/contracts"; import { applyOracleConsultationDirective } from "../../agent/oracle-consultation-policy.js"; +import { assignConfiguredOraclePolicyExperiment } from "../../agent/oracle-policy-experiment.js"; +import { + createRoutingReceipt, + resolveAgentProfileSelection, +} from "../../agent/routing-receipt.js"; import { isAssistantMessage } from "../../agent/type-guards.js"; import type { Attachment as AgentAttachment, @@ -45,6 +50,7 @@ import { } from "../../services/compliance/recorder.js"; import { recordIntelligentRouterChatMetric, + resolveIntelligentRouterProfileHint, selectIntelligentRouterModel, } from "../../services/intelligent-router/recorder.js"; import { recordAssistantUsageMetric } from "../../services/usage-analytics/recorder.js"; @@ -52,6 +58,7 @@ import { evaluateModelPolicy } from "../../services/workspace-config/policy.js"; import { toSessionModelMetadata } from "../../session/manager.js"; import { createRuntimeSessionSummaryUpdater } from "../../session/runtime-summary-updater.js"; import { recordSseSkip } from "../../telemetry.js"; +import { recordOraclePolicyExperimentAssignment } from "../../telemetry/oracle-policy.js"; import { createLogger } from "../../utils/logger.js"; import type { WebServerContext } from "../app-context.js"; import { @@ -321,6 +328,17 @@ export async function handleChat( // Resolve model from registry through the intelligent router. With no history, // this preserves the caller's requested model; with enough data or overrides, // it can select a better-scoring model and expose explicit fallbacks. + const profileSelection = resolveAgentProfileSelection({ + requestedProfile: resolveIntelligentRouterProfileHint(req, chatReq), + sessionPin: sessionManager.getHeader()?.agentProfilePin, + compatibilityProfile: "medium", + }); + if (chatReq.persistProfile && profileSelection.source === "request") { + sessionManager.updateAgentProfilePin({ + profile: profileSelection.requestedProfile, + updatedAt: new Date().toISOString(), + }); + } const routingSelection = selectIntelligentRouterModel({ req, requestedModel: resolveModelInputForRouting( @@ -328,9 +346,30 @@ export async function handleChat( defaultProvider, defaultModelId, ), - body: chatReq, + body: { ...chatReq, profile: profileSelection.requestedProfile }, }); + const oracleExperimentAssignment = assignConfiguredOraclePolicyExperiment( + sessionManager.getSessionId(), + ); + if (oracleExperimentAssignment) { + recordOraclePolicyExperimentAssignment({ + assignment: oracleExperimentAssignment, + sessionId: sessionManager.getSessionId(), + }); + } + const routedDecision = oracleExperimentAssignment + ? { + ...routingSelection.decision, + oracleConsultation: routingSelection.decision.oracleConsultation + ? { + ...routingSelection.decision.oracleConsultation, + policyVersion: oracleExperimentAssignment.policyVersion, + } + : undefined, + } + : routingSelection.decision; let registeredModel: RegisteredModel | undefined; + let selectedModelInputIndex = -1; let lastModelError: unknown; let selectedModelError: unknown; let selectedModelPolicyViolation: ReturnType< @@ -353,6 +392,7 @@ export async function handleChat( continue; } registeredModel = candidateModel; + selectedModelInputIndex = index; break; } catch (error) { lastModelError = error; @@ -378,6 +418,41 @@ export async function handleChat( if (!registeredModel) { throw selectedModelError ?? lastModelError; } + const usedFallback = selectedModelInputIndex > 0; + const routingReceipt = createRoutingReceipt( + { + ...routedDecision, + selectedModel: { + provider: registeredModel.provider, + model: registeredModel.id, + }, + }, + { + ...profileSelection, + ...(oracleExperimentAssignment + ? { + experiment: { + experimentId: oracleExperimentAssignment.experimentId, + arm: oracleExperimentAssignment.arm, + policyVersion: oracleExperimentAssignment.policyVersion, + }, + } + : {}), + ...(usedFallback || routingSelection.decision.reason !== "highest_score" + ? { + fallbackReason: usedFallback + ? "selected_model_unavailable_or_disallowed" + : routingSelection.decision.reason, + fallbackModel: usedFallback + ? { + provider: registeredModel.provider, + model: registeredModel.id, + } + : undefined, + } + : {}), + }, + ); const routeStartedAt = Date.now(); // Parse approval mode from request header (allows per-request override) @@ -442,10 +517,7 @@ export async function handleChat( : {}), }, ); - applyOracleConsultationDirective( - agent, - routingSelection.decision.oracleConsultation, - ); + applyOracleConsultationDirective(agent, routedDecision.oracleConsultation); // Hydrate conversation history (all messages except the current user input) const historyMessages = incomingMessages.slice(0, -1); @@ -487,6 +559,7 @@ export async function handleChat( }, { requestId, modelKey }, ); + sseSession.sendRoutingReceipt(routingReceipt); sseSession.startHeartbeat(); // Keep connection alive during idle periods // Track cleanup state to prevent double-cleanup @@ -785,7 +858,10 @@ export async function handleChat( // Handle message completion - persist to session if (event.type === "message_end") { - sessionManager.saveMessage(event.message); + const persistedMessage = isAssistantMessage(event.message) + ? { ...event.message, routingReceipt } + : event.message; + sessionManager.saveMessage(persistedMessage); if (isAssistantMessage(event.message)) { recordAssistantUsageMetric({ req, diff --git a/src/server/handlers/package.ts b/src/server/handlers/package.ts index 20df8bf8d..7f9e0210e 100644 --- a/src/server/handlers/package.ts +++ b/src/server/handlers/package.ts @@ -75,6 +75,7 @@ function serializeInspection(inspected: InspectedPackage) { : null, resources: inspected.resources ? { + agents: inspected.resources.agents, extensions: inspected.resources.extensions, skills: inspected.resources.skills, prompts: inspected.resources.prompts, diff --git a/src/server/hosted-session-manager.ts b/src/server/hosted-session-manager.ts index bbea25949..7ebae050d 100644 --- a/src/server/hosted-session-manager.ts +++ b/src/server/hosted-session-manager.ts @@ -1,4 +1,5 @@ import { randomUUID } from "node:crypto"; +import type { AgentProfilePin } from "@evalops/contracts"; import { and, asc, desc, eq, gte, isNull, sql } from "drizzle-orm"; import { isToolResultMessage } from "../agent/type-guards.js"; import type { AgentState, AppMessage } from "../agent/types.js"; @@ -93,6 +94,7 @@ export class HostedSessionManager { private writeError: unknown; private snapshot?: AgentState; private lastModelMetadata?: SessionModelMetadata; + private pendingAgentProfilePin?: AgentProfilePin; constructor(options: { scope: string; @@ -671,6 +673,7 @@ export class HostedSessionManager { promptMetadata: state.promptMetadata, promptContextManifest: getPersistedSessionPromptContextManifest(state), unifiedContextManifest: state.unifiedContextManifest, + agentProfilePin: this.pendingAgentProfilePin, systemPromptSourcePaths: state.systemPromptSourcePaths && state.systemPromptSourcePaths.length > 0 @@ -715,6 +718,36 @@ export class HostedSessionManager { } } + updateAgentProfilePin(pin: AgentProfilePin): boolean { + const profile = pin.profile.trim(); + if (!profile || !pin.updatedAt.trim()) return false; + const normalized = Object.freeze({ profile, updatedAt: pin.updatedAt }); + this.pendingAgentProfilePin = normalized; + if (!this.sessionInitialized) return true; + const headerIndex = this.entries.findIndex( + (entry) => entry.type === "session", + ); + if (headerIndex < 0) return false; + const entry = { + ...(this.entries[headerIndex] as SessionHeaderEntry), + agentProfilePin: normalized, + }; + this.entries[headerIndex] = entry; + const sessionId = this.sessionId; + this.enqueue(async () => { + await getDb() + .update(hostedSessionEntries) + .set({ entry }) + .where( + and( + eq(hostedSessionEntries.sessionId, sessionId), + eq(hostedSessionEntries.entryType, "session"), + ), + ); + }); + return true; + } + saveMessage(message: AppMessage): void { if ( isToolResultMessage(message) && diff --git a/src/server/session-serialization.ts b/src/server/session-serialization.ts index 072b4ab85..3addf924f 100644 --- a/src/server/session-serialization.ts +++ b/src/server/session-serialization.ts @@ -460,6 +460,7 @@ export function convertAppMessageToComposer( provider: message.provider, api: message.api, model: message.model, + routingReceipt: message.routingReceipt, }; } @@ -662,6 +663,7 @@ export function convertComposerMessageToApp( : createEmptyUsage(), stopReason: "stop", timestamp: toTimestamp(message.timestamp, context), + routingReceipt: message.routingReceipt, }; const results: AppMessage[] = [assistantMessage]; @@ -735,6 +737,7 @@ export function convertComposerMessageToApp( : createEmptyUsage(), stopReason: "stop", timestamp: toTimestamp(message.timestamp, context), + routingReceipt: message.routingReceipt, }; const results: AppMessage[] = [assistantMessage]; diff --git a/src/server/sse-session.ts b/src/server/sse-session.ts index 54109df07..2b57ba77d 100644 --- a/src/server/sse-session.ts +++ b/src/server/sse-session.ts @@ -1,4 +1,5 @@ import type { IncomingMessage, ServerResponse } from "node:http"; +import type { RoutingReceipt } from "@evalops/contracts"; import type { AgentEvent } from "../agent/types.js"; export interface SseContext { @@ -83,6 +84,12 @@ export class SseSession { this.write(`data: ${JSON.stringify(payload)}\n\n`); } + sendRoutingReceipt(receipt: RoutingReceipt): void { + this.write( + `data: ${JSON.stringify({ type: "routing_receipt", receipt })}\n\n`, + ); + } + sendHeartbeat(): void { this.write('data: {"type":"heartbeat"}\n\n'); } diff --git a/src/services/image-providers/cost.ts b/src/services/image-providers/cost.ts new file mode 100644 index 000000000..a1d8220f3 --- /dev/null +++ b/src/services/image-providers/cost.ts @@ -0,0 +1,208 @@ +/** + * Painter cost estimation and spend ceiling. + * + * Image generation is metered per-image and prices are not exposed on the API + * response in a stable field, so pre-call gating needs a price table. We do + * NOT ship fabricated prices: the default table is empty and the ceiling only + * enforces when an operator configures both `MAESTRO_PAINTER_PRICE_TABLE` and + * `MAESTRO_PAINTER_MAX_COST_CENTS`. When prices are unknown, the gate fails + * OPEN (allows the call) and surfaces a reason, rather than blocking work on + * missing data. + * + * Price table shape (JSON env): keyed by `${model}|${size}|${quality}` → + * whole cents per image. A `*` wildcard is allowed for size and quality, e.g. + * `{"gpt-image-2|*|high": 8}`. Lookups prefer the most specific key. + * + * @module services/image-providers/cost + */ + +import { createLogger } from "../../utils/logger.js"; + +const logger = createLogger("painter:cost"); + +export interface CostInput { + model: string; + size?: string; + quality?: string; + n?: number; +} + +export interface BudgetDecision { + ok: boolean; + /** True when a ceiling is configured and prices are known. */ + enforced: boolean; + estimatedCents?: number; + cumulativeCents: number; + ceilingCents?: number; + reason?: string; +} + +export interface PriceTable { + /** key `${model}|${size}|${quality}` → cents per image */ + [key: string]: number; +} + +function priceTableFromEnv(): PriceTable { + const raw = process.env.MAESTRO_PAINTER_PRICE_TABLE?.trim(); + if (!raw) return {}; + try { + const parsed = JSON.parse(raw); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + const table: PriceTable = {}; + for (const [k, v] of Object.entries(parsed)) { + if (typeof v === "number" && Number.isFinite(v) && v >= 0) { + table[k] = Math.floor(v); + } + } + return table; + } + logger.warn("MAESTRO_PAINTER_PRICE_TABLE is not a JSON object; ignoring"); + } catch { + logger.warn("MAESTRO_PAINTER_PRICE_TABLE is not valid JSON; ignoring"); + } + return {}; +} + +function lookupCents( + table: PriceTable, + model: string, + size: string, + quality: string, +): number | undefined { + const keys = [ + `${model}|${size}|${quality}`, + `${model}|${size}|*`, + `${model}|*|${quality}`, + `${model}|*|*`, + ]; + for (const k of keys) { + if (k in table) return table[k]; + } + return undefined; +} + +/** + * Estimate the cost in whole cents for a single image-API request. Returns + * `undefined` when no matching price is configured. + */ +export function estimateImageCost(input: CostInput): number | undefined { + const table = getPriceTable(); + const size = (input.size ?? "auto").toString(); + const quality = (input.quality ?? "auto").toString(); + const per = lookupCents(table, input.model, size, quality); + if (per === undefined) return undefined; + return per * Math.max(1, input.n ?? 1); +} + +/** + * Cached parsed price table, invalidated when MAESTRO_PAINTER_PRICE_TABLE + * changes. Env is immutable over a process lifetime in production, so this + * JSON.parses at most once; tests that mutate the env still re-parse because + * the cache key is the raw env string. + */ +let priceTableKey: string | undefined; +let priceTableCache: PriceTable = {}; + +function getPriceTable(): PriceTable { + const key = process.env.MAESTRO_PAINTER_PRICE_TABLE; + if (key !== priceTableKey) { + priceTableCache = priceTableFromEnv(); + priceTableKey = key; + } + return priceTableCache; +} + +/** + * Process-level spend accumulator with an optional ceiling. Singleton so + * multiple painter calls within one agent process share a budget. Reset + * between sessions by the caller (or by process restart). + */ +export class PainterBudget { + private spent = 0; + + /** Whole cents; undefined when no ceiling is configured. */ + readonly ceilingCents: number | undefined; + + constructor() { + const raw = process.env.MAESTRO_PAINTER_MAX_COST_CENTS?.trim(); + if (raw) { + const parsed = Number.parseInt(raw, 10); + this.ceilingCents = + Number.isFinite(parsed) && parsed >= 0 ? parsed : undefined; + } + } + + reset(): void { + this.spent = 0; + } + + /** + * Release previously reserved cents, e.g. when the API call failed and the + * provider did not charge. Clamped at zero so a buggy caller can't go + * negative and effectively raise the ceiling. + */ + release(cents: number): void { + if (cents > 0) this.spent = Math.max(0, this.spent - cents); + } + + get cumulativeCents(): number { + return this.spent; + } + + /** + * Check the request against the ceiling and reserve the estimate when + * allowed. Fails open when the ceiling or prices are unknown. + */ + checkAndReserve(input: CostInput): BudgetDecision { + const ceiling = this.ceilingCents; + const estimate = estimateImageCost(input); + + if (ceiling === undefined) { + return { ok: true, enforced: false, cumulativeCents: this.spent }; + } + + if (estimate === undefined) { + return { + ok: true, + enforced: false, + cumulativeCents: this.spent, + ceilingCents: ceiling, + reason: + "cost ceiling set but no price configured for this model/size/quality; gate disabled", + }; + } + + if (this.spent + estimate > ceiling) { + return { + ok: false, + enforced: true, + estimatedCents: estimate, + cumulativeCents: this.spent, + ceilingCents: ceiling, + reason: `painter cost ceiling would be exceeded (${this.spent} + ${estimate} > ${ceiling} cents)`, + }; + } + + this.spent += estimate; + return { + ok: true, + enforced: true, + estimatedCents: estimate, + cumulativeCents: this.spent, + ceilingCents: ceiling, + }; + } +} + +let singleton: PainterBudget | null = null; + +/** Process-wide budget. Re-evaluates the ceiling env on first access only. */ +export function getPainterBudget(): PainterBudget { + if (!singleton) singleton = new PainterBudget(); + return singleton; +} + +/** Test-only: discard the cached budget so env changes take effect. */ +export function resetPainterBudgetSingleton(): void { + singleton = null; +} diff --git a/src/services/image-providers/flux.ts b/src/services/image-providers/flux.ts new file mode 100644 index 000000000..5fbe0af7d --- /dev/null +++ b/src/services/image-providers/flux.ts @@ -0,0 +1,143 @@ +import type { GenerateOptions, ImageProvider, ImageResult } from "./types.js"; + +/** + * FLUX image provider via fal.ai. + * + * fal.run FLUX is generate-only here. It returns image URLs (not base64), so + * the provider fetches each URL and hands the bytes to the same `writeImage` + * callback the OpenAI provider uses. Masked editing is not supported (FLUX + * inpainting is a separate endpoint on fal; left for a follow-up). + * + * Stub-tested only: there is no live FAL_KEY in this environment, so the + * request/response mapping is unit-tested with an injected transport. Verify + * against your fal.ai account before relying on it. + */ + +const DEFAULT_MODEL = "fal-ai/flux/schnell"; +const FAL_BASE = "https://fal.run"; + +interface FalImage { + url?: string; + content_type?: string; +} + +interface FalResponse { + images?: FalImage[]; +} + +export interface FluxTransport { + post(url: string, body: unknown, signal?: AbortSignal): Promise; + get(url: string, signal?: AbortSignal): Promise; +} + +export interface FluxImageProviderOptions { + apiKey: string; + model?: string; + writeImage: (bytes: Buffer, ext: string) => Promise<{ path: string }>; + /** Test seam; defaults to a fetch-backed transport. */ + transport?: FluxTransport; +} + +/** Map our size strings to fal's image_size enum. */ +export function mapSizeToFlux(size?: string): string { + switch (size) { + case "1024x1024": + return "square"; + case "1536x1024": + case "1792x1024": + return "landscape_4_3"; + case "1024x1536": + case "1024x1792": + return "portrait_4_3"; + default: + return "square_hd"; + } +} + +function defaultTransport(apiKey: string): FluxTransport { + const headers = { + Authorization: `Key ${apiKey}`, + "Content-Type": "application/json", + }; + return { + async post(url, body, signal) { + const res = await fetch(url, { + method: "POST", + headers, + body: JSON.stringify(body), + signal, + }); + if (!res.ok) { + throw new Error( + `fal.ai request failed (${res.status}): ${await res.text()}`, + ); + } + return (await res.json()) as FalResponse; + }, + async get(url, signal) { + const res = await fetch(url, { signal }); + if (!res.ok) { + throw new Error(`fal.ai image fetch failed (${res.status})`); + } + return Buffer.from(await res.arrayBuffer()); + }, + }; +} + +export class FluxImageProvider implements ImageProvider { + readonly id = "flux"; + readonly supports = { generate: true, edit: false, mask: false } as const; + + private readonly apiKey: string; + private readonly model: string; + private readonly writeImage: FluxImageProviderOptions["writeImage"]; + private readonly transport: FluxTransport; + + constructor(options: FluxImageProviderOptions) { + this.apiKey = options.apiKey; + this.model = options.model ?? DEFAULT_MODEL; + this.writeImage = options.writeImage; + this.transport = options.transport ?? defaultTransport(this.apiKey); + } + + async generate(options: GenerateOptions): Promise { + const body = { + prompt: options.prompt, + image_size: mapSizeToFlux(options.size), + num_images: Math.max(1, options.n ?? 1), + output_format: "png" as const, + enable_safety_checker: true, + }; + const response = await this.transport.post( + `${FAL_BASE}/${this.model}`, + body, + options.signal, + ); + + const items = response.images ?? []; + if (items.length === 0) { + throw new Error("FLUX provider returned no images."); + } + + const images = []; + for (const item of items) { + if (!item.url) continue; + const bytes = await this.transport.get(item.url, options.signal); + const { path } = await this.writeImage(bytes, "png"); + images.push({ + path, + mimeType: item.content_type ?? "image/png", + }); + } + if (images.length === 0) { + throw new Error("FLUX provider returned no downloadable image URLs."); + } + return { images, provider: this.id, model: this.model }; + } + + async edit(): Promise { + throw new Error( + "FLUX provider does not support editing (generate only). Use the OpenAI provider for edits.", + ); + } +} diff --git a/src/services/image-providers/index.ts b/src/services/image-providers/index.ts new file mode 100644 index 000000000..68329aecf --- /dev/null +++ b/src/services/image-providers/index.ts @@ -0,0 +1,53 @@ +import { FluxImageProvider } from "./flux.js"; +import { OpenAIImageProvider } from "./openai.js"; +import type { ImageProvider } from "./types.js"; + +export type { + ImageBackground, + ImageOutput, + ImageProvider, + ImageQuality, + ImageResult, + ImageSize, +} from "./types.js"; + +export type PainterProviderName = "openai" | "flux"; + +export interface CreateImageProviderOptions { + provider?: PainterProviderName; + apiKey?: string; + baseURL?: string; + model?: string; + /** + * Persist decoded image bytes to disk and return the absolute path. + * Forwarded to the provider so it stays free of filesystem policy. + */ + writeImage: (bytes: Buffer, ext: string) => Promise<{ path: string }>; +} + +/** + * Build the configured image provider. Selection via the `provider` field + * (the painter resolves this from MAESTRO_PAINTER_PROVIDER, default "openai"). + * Each provider declares its own `supports` map; the painter must respect it + * before calling generate/edit. + */ +export function createImageProvider( + options: CreateImageProviderOptions, +): ImageProvider { + if (options.provider === "flux") { + if (!options.apiKey) { + throw new Error("FLUX provider requires FAL_KEY to be set."); + } + return new FluxImageProvider({ + apiKey: options.apiKey, + model: options.model, + writeImage: options.writeImage, + }); + } + return new OpenAIImageProvider({ + apiKey: options.apiKey, + baseURL: options.baseURL, + model: options.model, + writeImage: options.writeImage, + }); +} diff --git a/src/services/image-providers/masks.ts b/src/services/image-providers/masks.ts new file mode 100644 index 000000000..6aaec9569 --- /dev/null +++ b/src/services/image-providers/masks.ts @@ -0,0 +1,161 @@ +/** + * Bounding-box → mask synthesis. + * + * The OpenAI image edit endpoint accepts a mask PNG whose transparent pixels + * mark the region the model is allowed to change. Requiring callers to author + * that PNG by hand is poor UX, so Painter accepts a pixel bounding box and we + * synthesize the mask here. + * + * This module is dependency-free (only `node:zlib`) so it is fully unit- + * testable and works whether or not the optional `sharp` package is present. + * Dimension detection of arbitrary input formats is handled by the caller + * (see `readPngDimensions` for the common PNG case). + * + * Convention: transparent (alpha 0) = editable region; opaque (alpha 255) = + * preserved. Matches the OpenAI Images API mask semantics. + * + * @module services/image-providers/masks + */ + +import { deflateSync } from "node:zlib"; + +/** Pixel rectangle. Coordinates outside the canvas are clipped to bounds. */ +export interface MaskRegion { + x: number; + y: number; + width: number; + height: number; +} + +export interface ImageDimensions { + width: number; + height: number; +} + +const PNG_SIGNATURE = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, +]); + +// --- CRC32 (table-based, standard IEEE 802.3 polynomial) --------------------- + +const CRC_TABLE: Uint32Array = (() => { + const table = new Uint32Array(256); + for (let n = 0; n < 256; n++) { + let c = n; + for (let k = 0; k < 8; k++) { + c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1; + } + table[n] = c >>> 0; + } + return table; +})(); + +export function crc32(buf: Buffer): number { + let c = 0xffffffff; + for (let i = 0; i < buf.length; i++) { + const idx = (c ^ (buf[i] ?? 0)) & 0xff; + c = (CRC_TABLE[idx] ?? 0) ^ (c >>> 8); + } + return (c ^ 0xffffffff) >>> 0; +} + +function pngChunk(type: string, data: Buffer): Buffer { + const typeBuf = Buffer.from(type, "ascii"); + const lenBuf = Buffer.alloc(4); + lenBuf.writeUInt32BE(data.length, 0); + const crcBuf = Buffer.alloc(4); + crcBuf.writeUInt32BE(crc32(Buffer.concat([typeBuf, data])), 0); + return Buffer.concat([lenBuf, typeBuf, data, crcBuf]); +} + +function ihdr(width: number, height: number): Buffer { + const data = Buffer.alloc(13); + data.writeUInt32BE(width, 0); + data.writeUInt32BE(height, 4); + // bit depth 8, color type 6 (RGBA), compression 0, filter 0, interlace 0 + data[8] = 8; + data[9] = 6; + data[10] = 0; + data[11] = 0; + data[12] = 0; + return data; +} + +/** Clip a region to the canvas, returning a normalized {x,y,width,height}. */ +export function clipRegion( + canvas: ImageDimensions, + region: MaskRegion, +): { x: number; y: number; width: number; height: number } { + const x = Math.max(0, Math.floor(region.x)); + const y = Math.max(0, Math.floor(region.y)); + const x2 = Math.min(canvas.width, Math.ceil(region.x + region.width)); + const y2 = Math.min(canvas.height, Math.ceil(region.y + region.height)); + return { x, y, width: Math.max(0, x2 - x), height: Math.max(0, y2 - y) }; +} + +/** + * Encode an RGBA mask PNG of the given canvas size. Pixels inside `region` + * are transparent (editable); everything else is opaque (preserved). + * + * @throws if canvas dimensions are non-positive or the region is degenerate + * after clipping. + */ +export function encodeMaskPng( + canvas: ImageDimensions, + region: MaskRegion, +): Buffer { + if (canvas.width <= 0 || canvas.height <= 0) { + throw new Error( + `encodeMaskPng: canvas dimensions must be positive (got ${canvas.width}x${canvas.height})`, + ); + } + const clipped = clipRegion(canvas, region); + if (clipped.width <= 0 || clipped.height <= 0) { + throw new Error( + "encodeMaskPng: region is empty after clipping to canvas bounds", + ); + } + + const { width, height } = canvas; + // Build raw scanlines: filter byte 0 (None) + width * 4 bytes RGBA. + const rowLen = width * 4; + const raw = Buffer.alloc((rowLen + 1) * height); + for (let y = 0; y < height; y++) { + const rowStart = y * (rowLen + 1); + raw[rowStart] = 0; // filter: None + const inY = y >= clipped.y && y < clipped.y + clipped.height; + for (let x = 0; x < width; x++) { + const px = rowStart + 1 + x * 4; + const inX = x >= clipped.x && x < clipped.x + clipped.width; + // RGB 0; alpha 0 inside region (editable), 255 outside (preserved). + raw[px] = 0; + raw[px + 1] = 0; + raw[px + 2] = 0; + raw[px + 3] = inX && inY ? 0 : 255; + } + } + + const idat = deflateSync(raw); + return Buffer.concat([ + PNG_SIGNATURE, + pngChunk("IHDR", ihdr(width, height)), + pngChunk("IDAT", idat), + pngChunk("IEND", Buffer.alloc(0)), + ]); +} + +/** + * Read {width, height} from a PNG buffer by parsing the IHDR chunk. + * Returns null if the buffer is not a PNG or the IHDR is malformed. + */ +export function readPngDimensions(buf: Buffer): ImageDimensions | null { + if (buf.length < 24) return null; + if (!PNG_SIGNATURE.equals(buf.subarray(0, 8))) return null; + // Bytes 12..16 = "IHDR"; 16..20 = width; 20..24 = height. + if (buf.toString("ascii", 12, 16) !== "IHDR") return null; + const width = buf.readUInt32BE(16); + const height = buf.readUInt32BE(20); + if (!Number.isFinite(width) || !Number.isFinite(height)) return null; + if (width <= 0 || height <= 0) return null; + return { width, height }; +} diff --git a/src/services/image-providers/openai.ts b/src/services/image-providers/openai.ts new file mode 100644 index 000000000..9391b39f5 --- /dev/null +++ b/src/services/image-providers/openai.ts @@ -0,0 +1,151 @@ +import { readFile } from "node:fs/promises"; +import { basename, extname } from "node:path"; +import OpenAI, { toFile } from "openai"; +import type { + EditOptions, + GenerateOptions, + ImageProvider, + ImageQuality, + ImageResult, + ImageSize, +} from "./types.js"; + +/** Default model. gpt-image-2 does NOT support transparency. */ +const DEFAULT_MODEL = "gpt-image-2"; + +interface RawImageItem { + b64_json?: string | null; + revised_prompt?: string | null; +} + +interface ImagesResponseLike { + data?: RawImageItem[]; +} + +function mimeTypeForPath(path: string): string { + switch (extname(path).toLowerCase()) { + case ".png": + return "image/png"; + case ".jpg": + case ".jpeg": + return "image/jpeg"; + case ".webp": + return "image/webp"; + default: + return "application/octet-stream"; + } +} + +/** Narrow the images.edit/generate response union to its non-streaming member. */ +function extractItems(response: unknown): RawImageItem[] { + if (response && typeof response === "object" && "data" in response) { + const data = (response as ImagesResponseLike).data; + return Array.isArray(data) ? data : []; + } + return []; +} + +export interface OpenAIImageProviderOptions { + apiKey?: string; + baseURL?: string; + model?: string; + /** + * Optional pre-built client, primarily for tests. When omitted the provider + * constructs `new OpenAI(...)` from `apiKey`/`baseURL`. + */ + client?: OpenAI; + /** + * Persist decoded image bytes to disk and return the absolute path. + * Kept as a callback so the provider stays free of filesystem policy. + */ + writeImage: (bytes: Buffer, ext: string) => Promise<{ path: string }>; +} + +export class OpenAIImageProvider implements ImageProvider { + readonly id = "openai"; + readonly supports = { generate: true, edit: true, mask: true } as const; + + private readonly client: OpenAI; + private readonly model: string; + private readonly writeImage: OpenAIImageProviderOptions["writeImage"]; + + constructor(options: OpenAIImageProviderOptions) { + this.client = + options.client ?? + new OpenAI({ apiKey: options.apiKey, baseURL: options.baseURL }); + this.model = options.model ?? DEFAULT_MODEL; + this.writeImage = options.writeImage; + } + + async generate(options: GenerateOptions): Promise { + const response = await this.client.images.generate( + { + model: this.model, + prompt: options.prompt, + size: options.size as ImageSize | undefined, + quality: options.quality as ImageQuality | undefined, + background: options.background, + n: options.n ?? 1, + }, + { signal: options.signal }, + ); + return this.toResult(extractItems(response)); + } + + async edit(options: EditOptions): Promise { + if (!this.supports.edit) { + throw new Error("OpenAI image provider does not support editing."); + } + if (options.images.length === 0) { + throw new Error("Edit mode requires at least one input image path."); + } + + const imageFiles = await Promise.all( + options.images.slice(0, 3).map(async (p) => + toFile(await readFile(p), basename(p), { + type: mimeTypeForPath(p), + }), + ), + ); + + const params: Parameters[0] = { + model: this.model, + image: imageFiles, + prompt: options.prompt, + size: options.size as Parameters[0]["size"], + quality: options.quality as ImageQuality | undefined, + n: options.n ?? 1, + }; + + if (options.maskPath) { + params.mask = await toFile( + await readFile(options.maskPath), + basename(options.maskPath), + { type: mimeTypeForPath(options.maskPath) }, + ); + } + + const response = await this.client.images.edit(params, { + signal: options.signal, + }); + return this.toResult(extractItems(response)); + } + + private async toResult(items: RawImageItem[]): Promise { + const images = []; + for (const item of items) { + if (!item.b64_json) continue; + const buf = Buffer.from(item.b64_json, "base64"); + const { path } = await this.writeImage(buf, "png"); + images.push({ + path, + mimeType: "image/png", + revisedPrompt: item.revised_prompt ?? undefined, + }); + } + if (images.length === 0) { + throw new Error("OpenAI image API returned no image data."); + } + return { images, provider: this.id, model: this.model }; + } +} diff --git a/src/services/image-providers/types.ts b/src/services/image-providers/types.ts new file mode 100644 index 000000000..a6113db5d --- /dev/null +++ b/src/services/image-providers/types.ts @@ -0,0 +1,74 @@ +/** + * Image provider abstraction. + * + * Image generation/editing is NOT symmetric across providers the way chat + * completion is: today only OpenAI's image API cleanly supports both + * generation and masked editing. The `supports` map keeps that asymmetry + * honest rather than pretending every provider implements every capability. + * + * Add a new provider by implementing `ImageProvider` and registering it in + * `./index.ts`. Do not weaken this interface to make a generate-only or + * edit-only provider look symmetric — declare what it actually supports and + * let callers fail closed. + * + * @module services/image-providers + */ + +export type ImageSize = + | "1024x1024" + | "1536x1024" + | "1024x1536" + | "1792x1024" + | "1024x1792" + | "auto"; + +export type ImageQuality = "low" | "medium" | "high" | "auto"; + +export type ImageBackground = "transparent" | "opaque" | "auto"; + +export interface GenerateOptions { + prompt: string; + size?: ImageSize; + quality?: ImageQuality; + background?: ImageBackground; + /** Number of images to produce. Defaults to 1. */ + n?: number; + signal?: AbortSignal; +} + +export interface EditOptions extends Omit { + /** 1-3 input image paths to edit. */ + images: string[]; + /** + * Optional mask image path. Transparent regions of the mask mark the + * areas the model is allowed to change. Omit for a whole-image edit. + */ + maskPath?: string; +} + +export interface ImageOutput { + /** Absolute path where the generated image was persisted. */ + path: string; + mimeType: string; + /** Provider-returned revised prompt, when available. */ + revisedPrompt?: string; +} + +export interface ImageResult { + images: ImageOutput[]; + /** Provider id (e.g. "openai"). */ + provider: string; + model: string; +} + +export interface ImageProvider { + id: string; + /** Capabilities this provider actually implements. Used to fail closed. */ + supports: { + generate: boolean; + edit: boolean; + mask: boolean; + }; + generate(options: GenerateOptions): Promise; + edit(options: EditOptions): Promise; +} diff --git a/src/services/intelligent-router/recorder.ts b/src/services/intelligent-router/recorder.ts index 4603a9b82..0b7cd8602 100644 --- a/src/services/intelligent-router/recorder.ts +++ b/src/services/intelligent-router/recorder.ts @@ -79,6 +79,23 @@ export function resolveIntelligentRouterStrategy( : undefined; } +export function resolveIntelligentRouterProfileHint( + req: IncomingMessage, + body?: unknown, +): string | undefined { + const header = firstHeader(req, [ + "x-maestro-agent-profile", + "x-composer-agent-profile", + ]); + if (header) return header; + if (!body || typeof body !== "object" || !("profile" in body)) + return undefined; + const profile = (body as { profile?: unknown }).profile; + return typeof profile === "string" && profile.trim() + ? profile.trim() + : undefined; +} + export function resolveIntelligentRouterTaskSummary( body: unknown, ): string | undefined { @@ -143,17 +160,10 @@ export function selectIntelligentRouterModel(params: { const taskType = resolveIntelligentRouterTaskType(params.req, params.body); const modelHint = params.requestedModel ?? undefined; const strategy = resolveIntelligentRouterStrategy(params.req); - const profileHint = - firstHeader(params.req, [ - "x-maestro-agent-profile", - "x-composer-agent-profile", - ]) ?? - (params.body && - typeof params.body === "object" && - "profile" in params.body && - typeof (params.body as { profile?: unknown }).profile === "string" - ? (params.body as { profile: string }).profile - : undefined); + const profileHint = resolveIntelligentRouterProfileHint( + params.req, + params.body, + ); const decision = getIntelligentRouterService().routeRequest({ taskType, taskSummary: resolveIntelligentRouterTaskSummary(params.body), diff --git a/src/session/manager.ts b/src/session/manager.ts index f0dda07a9..b7557d644 100644 --- a/src/session/manager.ts +++ b/src/session/manager.ts @@ -11,6 +11,7 @@ import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; import { extname, join, resolve } from "node:path"; +import type { AgentProfilePin } from "@evalops/contracts"; import { v4 as uuidv4 } from "uuid"; import { isToolResultMessage } from "../agent/type-guards.js"; import type { @@ -270,6 +271,7 @@ export class SessionManager { private leafId: string | null = null; private flushed = false; private _hasAssistantMessage = false; + private pendingAgentProfilePin?: AgentProfilePin; /** * Creates a new SessionManager. @@ -545,6 +547,8 @@ export class SessionManager { promptMetadata: state.promptMetadata, promptContextManifest: getPersistedSessionPromptContextManifest(state), unifiedContextManifest: state.unifiedContextManifest, + agentProfilePin: + this.pendingAgentProfilePin ?? provisionalHeader?.agentProfilePin, systemPromptSourcePaths: state.systemPromptSourcePaths && state.systemPromptSourcePaths.length > 0 @@ -626,6 +630,28 @@ export class SessionManager { return true; } + updateAgentProfilePin(pin: AgentProfilePin): boolean { + if (!this.enabled) return false; + const profile = pin.profile.trim(); + if (!profile || !pin.updatedAt.trim()) return false; + const normalized = Object.freeze({ profile, updatedAt: pin.updatedAt }); + this.pendingAgentProfilePin = normalized; + if (!this.sessionInitialized) return true; + const headerIndex = this.fileEntries.findIndex( + (entry) => entry.type === "session", + ); + if (headerIndex < 0) return false; + const entry = { + ...(this.fileEntries[headerIndex] as SessionHeaderEntry), + agentProfilePin: normalized, + }; + this.fileEntries[headerIndex] = entry; + this.metadataCache.apply(entry); + this.rewriteSessionFile(); + this.flushed = true; + return true; + } + private scheduleAutoPrune(): void { pendingAutoPruneManagers.add(this); registerAutoPruneBeforeExit(); diff --git a/src/session/types.ts b/src/session/types.ts index 79e290a6f..56f4b0d31 100644 --- a/src/session/types.ts +++ b/src/session/types.ts @@ -1,3 +1,4 @@ +import type { AgentProfilePin } from "@evalops/contracts"; import type { AppMessage, ImageContent, TextContent } from "../agent/types.js"; import type { PromptProjectDocManifest } from "../config/index.js"; import type { UnifiedContextManifest } from "../context/manifest-types.js"; @@ -47,6 +48,7 @@ export interface SessionHeaderEntry { // actually loaded, even if that file is later deleted or moved while a // session is paused. See #2602. systemPromptSourcePaths?: string[]; + agentProfilePin?: AgentProfilePin; } function resolveSessionPromptContextManifest( diff --git a/src/telemetry/oracle-policy.ts b/src/telemetry/oracle-policy.ts new file mode 100644 index 000000000..c86a62592 --- /dev/null +++ b/src/telemetry/oracle-policy.ts @@ -0,0 +1,21 @@ +import type { OraclePolicyExperimentAssignment } from "../agent/oracle-policy-experiment.js"; +import { recordTelemetry } from "../telemetry.js"; + +export function recordOraclePolicyExperimentAssignment(input: { + assignment: OraclePolicyExperimentAssignment; + sessionId: string; +}): void { + void recordTelemetry({ + type: "staged-rollout-surface", + timestamp: new Date().toISOString(), + event: "internal_gate_used", + surfaceId: "oracle-policy-experiment", + surfaceType: "internal_gate", + metadata: { + sessionId: input.sessionId, + "oracle.experiment_id": input.assignment.experimentId, + "oracle.arm": input.assignment.arm, + "oracle.policy_version": input.assignment.policyVersion, + }, + }); +} diff --git a/src/tools/index.ts b/src/tools/index.ts index aad666476..a1e0ecf45 100644 --- a/src/tools/index.ts +++ b/src/tools/index.ts @@ -31,6 +31,7 @@ * * **AI & Automation:** * - oracle: Sub-agent for complex reasoning + * - painter: Image generation/editing (requires OPENAI_API_KEY) * - todo: Task list management * - ask_user: Structured user prompts * @@ -84,6 +85,7 @@ import { searchTool } from "./search.js"; // AI and automation import { oracleTool } from "./oracle.js"; +import { painterTool } from "./painter.js"; import { statusTool } from "./status.js"; import { todoTool } from "./todo.js"; @@ -162,6 +164,7 @@ export const codingTools = [ readTool, listTool, oracleTool, + painterTool, findTool, extractDocumentTool, // Search capabilities @@ -207,6 +210,7 @@ export const toolRegistry: Record = { read: readTool, list: listTool, oracle: oracleTool, + painter: painterTool, find: findTool, extract_document: extractDocumentTool, search: searchTool, diff --git a/src/tools/lazy-registry.ts b/src/tools/lazy-registry.ts index b5a020e5b..a1bf03808 100644 --- a/src/tools/lazy-registry.ts +++ b/src/tools/lazy-registry.ts @@ -4,6 +4,7 @@ const lazyToolLoaders = { read: async () => (await import("./read.js")).readTool, list: async () => (await import("./list.js")).listTool, oracle: async () => (await import("./oracle.js")).oracleTool, + painter: async () => (await import("./painter.js")).painterTool, find: async () => (await import("./find.js")).findTool, extract_document: async () => (await import("./extract-document.js")).extractDocumentTool, diff --git a/src/tools/painter.ts b/src/tools/painter.ts new file mode 100644 index 000000000..f39b8d2a1 --- /dev/null +++ b/src/tools/painter.ts @@ -0,0 +1,393 @@ +import { randomBytes } from "node:crypto"; +import { mkdir, readFile, unlink, writeFile } from "node:fs/promises"; +import { homedir } from "node:os"; +import { join, resolve } from "node:path"; +import { Type } from "@sinclair/typebox"; +import { getPainterBudget } from "../services/image-providers/cost.js"; +import { createImageProvider } from "../services/image-providers/index.js"; +import { + type ImageDimensions, + type MaskRegion, + encodeMaskPng, + readPngDimensions, +} from "../services/image-providers/masks.js"; +import type { + ImageQuality, + ImageSize, +} from "../services/image-providers/types.js"; +import { createLogger } from "../utils/logger.js"; +import { getImageMetadata } from "./image-processor.js"; +import { createTool } from "./tool-dsl.js"; + +const logger = createLogger("tools:painter"); + +const DEFAULT_PAINTER_TIMEOUT_MS = 180_000; +const DEFAULT_MODEL = "gpt-image-2"; + +export function painterTimeoutMs(): number { + const raw = process.env.MAESTRO_PAINTER_TIMEOUT_MS; + if (!raw) return DEFAULT_PAINTER_TIMEOUT_MS; + const parsed = Number.parseInt(raw, 10); + return Number.isFinite(parsed) && parsed > 0 + ? parsed + : DEFAULT_PAINTER_TIMEOUT_MS; +} + +export function painterOutputDir(): string { + const override = process.env.MAESTRO_PAINTER_OUTPUT_DIR?.trim(); + if (override) return resolve(override); + return join(homedir(), ".maestro", "assets", "painter"); +} + +export function resolvePainterModel(): string { + return process.env.MAESTRO_PAINTER_MODEL?.trim() || DEFAULT_MODEL; +} + +/** Resolve the image provider. "openai" (default) or "flux" via fal.ai. */ +export function resolvePainterProvider(): "openai" | "flux" { + const raw = process.env.MAESTRO_PAINTER_PROVIDER?.trim().toLowerCase(); + return raw === "flux" ? "flux" : "openai"; +} + +export async function persistImage( + bytes: Buffer, + ext: string, +): Promise<{ path: string }> { + const dir = painterOutputDir(); + await mkdir(dir, { recursive: true }); + const name = `painter-${Date.now()}-${randomBytes(4).toString("hex")}.${ext}`; + const path = join(dir, name); + await writeFile(path, bytes); + return { path }; +} + +/** Errno-style codes that indicate a transient network fault worth retrying. */ +const TRANSIENT_ERRNO_CODES = new Set([ + "ETIMEDOUT", + "ECONNRESET", + "ENOTFOUND", + "ECONNREFUSED", + "EAI_AGAIN", + "EPIPE", + "EHOSTUNREACH", + "ENETUNREACH", +]); + +/** + * Classify whether an image-API failure is worth one retry. Image generation + * is expensive, so we retry only on clearly transient network/rate-limit + * faults and never on validation or content-policy rejections. + */ +export function isTransientImageError(err: unknown): boolean { + const code = (err as { code?: unknown })?.code; + if ( + typeof code === "string" && + TRANSIENT_ERRNO_CODES.has(code.toUpperCase()) + ) { + return true; + } + const msg = err instanceof Error ? err.message : String(err); + return /rate limit|too many requests|timeout|timed out|econnreset|enotfound|etimedout|econnrefused|service unavailable|bad gateway|5\d{2}/i.test( + msg, + ); +} + +export type MaskDimensionSource = "png-header" | "sharp" | "explicit"; + +/** + * Resolve a pixel bounding box into a mask file path, writing the synthesized + * mask into the painter output directory. Dimensions come from (in order): + * the PNG header (no deps), the optional `sharp` package, or an explicit + * caller-supplied size. Returns the temp mask path plus how dims were found. + */ +export async function buildMaskPath( + images: string[], + region: MaskRegion, + explicitSize?: ImageDimensions, +): Promise<{ + maskPath: string; + dimensions: ImageDimensions; + source: MaskDimensionSource; +}> { + if (images.length === 0) { + throw new Error("mask synthesis requires at least one input image path."); + } + const inputPath = images[0]; + if (!inputPath) { + throw new Error("mask synthesis received an undefined input image path."); + } + const buf = await readFile(inputPath); + + let dimensions: ImageDimensions | null = readPngDimensions(buf); + let source: MaskDimensionSource = "png-header"; + + if (!dimensions) { + if (explicitSize) { + dimensions = explicitSize; + source = "explicit"; + } else { + const meta = await getImageMetadata(buf); + if (meta?.width && meta.height) { + dimensions = { width: meta.width, height: meta.height }; + source = "sharp"; + } + } + } + + if (!dimensions) { + throw new Error( + "Could not determine input image dimensions for mask synthesis. Use a PNG input, enable Sharp, or pass maskSize explicitly.", + ); + } + + const maskBuf = encodeMaskPng(dimensions, region); + const dir = painterOutputDir(); + await mkdir(dir, { recursive: true }); + const maskPath = join( + dir, + `mask-${Date.now()}-${randomBytes(4).toString("hex")}.png`, + ); + await writeFile(maskPath, maskBuf); + return { maskPath, dimensions, source }; +} + +const painterSchema = Type.Object({ + mode: Type.Union([Type.Literal("generate"), Type.Literal("edit")], { + description: + "generate = create a new image from a text prompt. edit = modify one or more existing images using a prompt (and optional mask).", + }), + prompt: Type.String({ + description: + "The image prompt. For edit mode, describe the desired change. Be concrete about subject, style, composition, and any text to render.", + }), + images: Type.Optional( + Type.Array(Type.String(), { + description: + "Edit mode only. 1-3 absolute or workspace-relative paths to input images to edit.", + }), + ), + mask: Type.Optional( + Type.String({ + description: + "Edit mode only. Path to a mask image; transparent regions mark areas the model may change. Omit to edit the whole image. Mutually exclusive with maskRegion (which synthesizes the mask for you).", + }), + ), + maskRegion: Type.Optional( + Type.Object( + { + x: Type.Number({ description: "Region left edge, in pixels." }), + y: Type.Number({ description: "Region top edge, in pixels." }), + width: Type.Number({ description: "Region width, in pixels." }), + height: Type.Number({ description: "Region height, in pixels." }), + }, + { + description: + "Edit mode only. A pixel bounding box; the mask is synthesized automatically (transparent = editable). Coordinates outside the image are clipped.", + }, + ), + ), + maskSize: Type.Optional( + Type.Object( + { + width: Type.Integer({ minimum: 1 }), + height: Type.Integer({ minimum: 1 }), + }, + { + description: + "Explicit input image dimensions, used only for mask synthesis when the input is not a PNG and Sharp is unavailable. Optional.", + }, + ), + ), + size: Type.Optional( + Type.String({ + description: + "Output size. Standard: 1024x1024, 1536x1024, 1024x1536, 1792x1024, 1024x1792, or auto. gpt-image-2 also accepts arbitrary WxH divisible by 16 with aspect ratio 1:3 to 3:1.", + }), + ), + quality: Type.Optional( + Type.Union( + [ + Type.Literal("low"), + Type.Literal("medium"), + Type.Literal("high"), + Type.Literal("auto"), + ], + { + description: + "Output quality. Higher quality is slower and more expensive. Defaults to auto.", + }, + ), + ), + n: Type.Optional( + Type.Integer({ + description: "Number of images to produce (1-4). Defaults to 1.", + minimum: 1, + maximum: 4, + }), + ), +}); + +export interface PainterToolDetails { + provider: string; + model: string; + mode: "generate" | "edit"; + paths: string[]; +} + +export const painterTool = createTool( + { + name: "painter", + description: + "Generate or edit images via an image model (default gpt-image-2). Use for UI mockups, app icons, illustrations, and editing existing images (e.g. redacting a screenshot). Outputs are persisted to disk and returned as absolute paths; reference them by path in later turns. Requires OPENAI_API_KEY.", + schema: painterSchema, + maxRetries: 1, + retryDelayMs: 2000, + shouldRetry: isTransientImageError, + getToolUseSummary: (params) => `painter ${params.mode}`, + getActivityDescription: (params) => + params.mode === "edit" ? "Editing image" : "Generating image", + async run(params, { respond, signal }) { + const providerName = resolvePainterProvider(); + const apiKeyEnv = providerName === "flux" ? "FAL_KEY" : "OPENAI_API_KEY"; + const apiKey = process.env[apiKeyEnv]; + if (!apiKey) { + throw new Error( + `Painter provider "${providerName}" requires ${apiKeyEnv} to be set in the environment where the agent runs.`, + ); + } + + if ( + params.mode === "edit" && + (!params.images || params.images.length === 0) + ) { + throw new Error( + "Painter edit mode requires at least one input image path (the `images` parameter).", + ); + } + + const budget = getPainterBudget().checkAndReserve({ + model: resolvePainterModel(), + size: params.size, + quality: params.quality, + n: params.n, + }); + if (!budget.ok) { + throw new Error(budget.reason ?? "painter cost ceiling exceeded"); + } + if (budget.enforced) { + logger.info("painter budget", { + estimated: budget.estimatedCents, + cumulative: budget.cumulativeCents, + ceiling: budget.ceilingCents, + }); + } + + const provider = createImageProvider({ + provider: providerName, + apiKey, + baseURL: process.env.MAESTRO_PAINTER_BASE_URL?.trim() || undefined, + model: resolvePainterModel(), + writeImage: persistImage, + }); + + if (!provider.supports[params.mode]) { + throw new Error( + params.mode === "edit" + ? `Painter provider "${provider.id}" does not support editing. Use the OpenAI provider (MAESTRO_PAINTER_PROVIDER=openai) for edits.` + : `Painter provider "${provider.id}" does not support generation.`, + ); + } + + // Combine the tool's abort signal with a hard timeout, without relying + // on AbortSignal.any so this works on any lib target. + const timeoutCtl = new AbortController(); + const onCallerAbort = () => timeoutCtl.abort(); + if (signal) { + if (signal.aborted) { + timeoutCtl.abort(); + } else { + signal.addEventListener("abort", onCallerAbort, { once: true }); + } + } + const timer = setTimeout(() => timeoutCtl.abort(), painterTimeoutMs()); + + let synthesizedMask: string | null = null; + try { + const shared = { + prompt: params.prompt, + size: params.size as ImageSize | undefined, + quality: params.quality as ImageQuality | undefined, + n: params.n, + signal: timeoutCtl.signal, + }; + + let maskPath = params.mask; + if (params.mode === "edit" && params.maskRegion) { + const built = await buildMaskPath( + params.images ?? [], + params.maskRegion, + params.maskSize, + ); + maskPath = built.maskPath; + synthesizedMask = built.maskPath; + logger.info("synthesized edit mask", { + source: built.source, + ...built.dimensions, + }); + } + + const result = + params.mode === "edit" + ? await provider.edit({ + ...shared, + images: params.images ?? [], + maskPath, + }) + : await provider.generate(shared); + + const lines = result.images.map( + (img, i) => + ` ${i + 1}. ${img.path}${img.revisedPrompt ? `\n revised prompt: ${img.revisedPrompt}` : ""}`, + ); + + logger.info("painter produced images", { + mode: params.mode, + model: result.model, + count: result.images.length, + }); + + return respond + .text( + [ + `Painter (${result.provider}/${result.model}) produced ${result.images.length} image(s):`, + ...lines, + "Outputs are on disk. Reference them by the absolute paths above.", + ].join("\n"), + ) + .detail({ + provider: result.provider, + model: result.model, + mode: params.mode, + paths: result.images.map((i) => i.path), + }); + } catch (err) { + // The provider did not produce images (transient retry exhausted, + // timeout, content-policy, mask failure, unsupported mode). Release + // the reserved estimate so failed calls don't permanently consume + // the spend ceiling — the provider did not charge for these. + if (budget.enforced && budget.estimatedCents) { + getPainterBudget().release(budget.estimatedCents); + } + throw err; + } finally { + clearTimeout(timer); + signal?.removeEventListener("abort", onCallerAbort); + // Best-effort cleanup of the synthesized mask; user-supplied mask + // paths are left untouched. + if (synthesizedMask) { + await unlink(synthesizedMask).catch(() => {}); + } + } + }, + }, +); diff --git a/src/utils/terminal-image.ts b/src/utils/terminal-image.ts new file mode 100644 index 000000000..57efde155 --- /dev/null +++ b/src/utils/terminal-image.ts @@ -0,0 +1,127 @@ +/** + * Terminal inline-image support detection and escape-sequence emission. + * + * Capable terminals (iTerm2, WezTerm, kitty, and sixel-aware terminals) can + * render image bytes sent via control sequences. This module detects support + * from the environment and encodes a PNG buffer for the matching protocol. + * + * Sixel is detected but not encoded here: sixel requires rasterizing the + * source image, which needs a real image library. iTerm2 (OSC 1337) and kitty + * (graphics protocol) both accept the raw image bytes base64-encoded, so they + * are fully implemented. + * + * IMPORTANT: these sequences must be written to a display-only stream (or a + * TUI render path that strips them before persistence). Tool-result content + * channels feed the model context as base64, so do NOT route inline escapes + * through tool text/image content — that wastes tokens and confuses the model. + * + * @module utils/terminal-image + */ + +export type TerminalImageSupport = "iterm" | "kitty" | "sixel" | "none"; + +export interface DetectEnv { + TERM_PROGRAM?: string; + TERM?: string; + COLORTERM?: string; +} + +/** + * Detect the best inline-image protocol the current terminal claims to support. + * Conservative: only returns a protocol when the environment explicitly + * advertises it. + */ +export function detectTerminalImageSupport( + env: DetectEnv = process.env, +): TerminalImageSupport { + const termProgram = env.TERM_PROGRAM ?? ""; + const term = env.TERM ?? ""; + + if (termProgram === "iTerm.app" || termProgram === "WezTerm") { + return "iterm"; + } + if (term === "xterm-kitty" || termProgram === "kitty") { + return "kitty"; + } + // Sixel: some terminals set COLORTERM or accept DECSCUSR queries; we only + // treat an explicit TERM containing "sixel" as a positive signal. + if (term.toLowerCase().includes("sixel")) { + return "sixel"; + } + return "none"; +} + +export interface InlineImageOptions { + /** Display width: pixels, "auto", or N cells (e.g. "40"). Default "auto". */ + width?: string | number; + /** Display height: pixels, "auto", or N cells. Default "auto". */ + height?: string | number; + /** Optional filename hint for iTerm2. */ + name?: string; +} + +const KITTY_CHUNK_SIZE = 4096; + +function b64(buf: Buffer): string { + return buf.toString("base64"); +} + +/** Encode a PNG for iTerm2's OSC 1337 inline-image protocol. */ +export function encodeItermInline( + buf: Buffer, + opts: InlineImageOptions = {}, +): string { + const width = opts.width ?? "auto"; + const height = opts.height ?? "auto"; + const args = [ + "inline=1", + `width=${width}`, + `height=${height}`, + "preserveAspectRatio=0", + ]; + if (opts.name) { + args.push(`name=${b64(Buffer.from(opts.name, "utf8"))}`); + } + return `\x1b]1337;File=${args.join(";")}:${b64(buf)}\x07`; +} + +/** + * Encode a PNG for kitty's graphics protocol, splitting the base64 payload + * into <=4096-byte chunks with `m=1` continuation markers. + */ +export function encodeKittyInline(buf: Buffer): string { + const payload = b64(buf); + const chunks: string[] = []; + for (let i = 0; i < payload.length; i += KITTY_CHUNK_SIZE) { + chunks.push(payload.slice(i, i + KITTY_CHUNK_SIZE)); + } + if (chunks.length === 0) { + // Empty image: still emit a single transmit chunk. + return "\x1b_Ga=T,f=100,t=f;\x1b\\"; + } + const out: string[] = []; + out.push(`\x1b_Ga=T,f=100,t=f;${chunks[0]}\x1b\\`); + for (let i = 1; i < chunks.length; i++) { + out.push(`\x1b_Gm=1;${chunks[i]}\x1b\\`); + } + return out.join(""); +} + +/** + * Encode an image for the detected protocol. Returns "" when the protocol is + * unsupported (including sixel, which is detected but not rasterized here). + */ +export function encodeInlineImage( + buf: Buffer, + support: TerminalImageSupport = detectTerminalImageSupport(), + opts: InlineImageOptions = {}, +): string { + switch (support) { + case "iterm": + return encodeItermInline(buf, opts); + case "kitty": + return encodeKittyInline(buf); + default: + return ""; + } +} diff --git a/test/agent/oracle-consultation-policy.test.ts b/test/agent/oracle-consultation-policy.test.ts index 30f53d7ae..433844c0c 100644 --- a/test/agent/oracle-consultation-policy.test.ts +++ b/test/agent/oracle-consultation-policy.test.ts @@ -116,6 +116,16 @@ describe("oracle consultation policy eval matrix", () => { expect(directive).toContain(ORACLE_CONSULTATION_POLICY_VERSION); }); + it("projects the assigned experiment policy version", () => { + expect( + recommendOracleConsultation({ + profileLevel: "high", + taskType: "coding", + policyVersion: "oracle-v2", + }).policyVersion, + ).toBe("oracle-v2"); + }); + it("queues recommended guidance exactly once for the next run", () => { const additions: string[] = []; const queued = applyOracleConsultationDirective( diff --git a/test/agent/oracle-policy-experiment.test.ts b/test/agent/oracle-policy-experiment.test.ts new file mode 100644 index 000000000..b3910e4f3 --- /dev/null +++ b/test/agent/oracle-policy-experiment.test.ts @@ -0,0 +1,80 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + assignConfiguredOraclePolicyExperiment, + assignOraclePolicyExperiment, + getOraclePolicyExperimentConfig, +} from "../../src/agent/oracle-policy-experiment.js"; + +const baseInput = { + experimentId: "oracle-policy-2026-07", + sessionId: "session-42", + allocation: 0.5, + controlVersion: "oracle-v1", + treatmentVersion: "oracle-v2", +}; + +describe("assignOraclePolicyExperiment", () => { + afterEach(() => vi.unstubAllEnvs()); + + it("resolves complete host configuration and rejects partial configuration", () => { + expect(getOraclePolicyExperimentConfig()).toBeUndefined(); + vi.stubEnv("MAESTRO_ORACLE_EXPERIMENT_ID", "oracle-july"); + expect(() => getOraclePolicyExperimentConfig()).toThrow( + /requires allocation/i, + ); + vi.stubEnv("MAESTRO_ORACLE_EXPERIMENT_ALLOCATION", "1"); + vi.stubEnv("MAESTRO_ORACLE_EXPERIMENT_CONTROL_VERSION", "oracle-v1"); + vi.stubEnv("MAESTRO_ORACLE_EXPERIMENT_TREATMENT_VERSION", "oracle-v2"); + expect(assignConfiguredOraclePolicyExperiment("session-a")).toMatchObject({ + experimentId: "oracle-july", + arm: "treatment", + policyVersion: "oracle-v2", + }); + }); + + it("returns the same assignment for the same experiment and session", () => { + const first = assignOraclePolicyExperiment(baseInput); + const second = assignOraclePolicyExperiment(baseInput); + + expect(second).toEqual(first); + expect(first.bucket).toBeGreaterThanOrEqual(0); + expect(first.bucket).toBeLessThan(1); + }); + + it("isolates assignments between experiments", () => { + const first = assignOraclePolicyExperiment(baseInput); + const second = assignOraclePolicyExperiment({ + ...baseInput, + experimentId: "oracle-policy-2026-08", + }); + + expect(second.bucket).not.toBe(first.bucket); + }); + + it("respects the allocation boundaries", () => { + expect( + assignOraclePolicyExperiment({ ...baseInput, allocation: 0 }).arm, + ).toBe("control"); + expect( + assignOraclePolicyExperiment({ ...baseInput, allocation: 1 }).arm, + ).toBe("treatment"); + }); + + it("projects the policy version selected by the assigned arm", () => { + expect( + assignOraclePolicyExperiment({ ...baseInput, allocation: 0 }), + ).toMatchObject({ arm: "control", policyVersion: "oracle-v1" }); + expect( + assignOraclePolicyExperiment({ ...baseInput, allocation: 1 }), + ).toMatchObject({ arm: "treatment", policyVersion: "oracle-v2" }); + }); + + it("rejects invalid allocations", () => { + expect(() => + assignOraclePolicyExperiment({ ...baseInput, allocation: -0.1 }), + ).toThrow(/allocation/i); + expect(() => + assignOraclePolicyExperiment({ ...baseInput, allocation: 1.1 }), + ).toThrow(/allocation/i); + }); +}); diff --git a/test/agent/oracle-policy-rollout.test.ts b/test/agent/oracle-policy-rollout.test.ts new file mode 100644 index 000000000..59927a1f9 --- /dev/null +++ b/test/agent/oracle-policy-rollout.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from "vitest"; +import { evaluateOraclePolicyRollout } from "../../src/agent/oracle-policy-rollout.js"; + +const baseInput = { + experimentId: "oracle-policy-2026-07", + expectedControlVersion: "oracle-v1", + expectedTreatmentVersion: "oracle-v2", + control: { + policyVersion: "oracle-v1", + verifiedSamples: 100, + successes: 80, + averageCostUsd: 1, + averageLatencyMs: 1_000, + safetyViolations: 0, + }, + treatment: { + policyVersion: "oracle-v2", + verifiedSamples: 100, + successes: 82, + averageCostUsd: 1.05, + averageLatencyMs: 1_050, + safetyViolations: 0, + }, +}; + +describe("evaluateOraclePolicyRollout", () => { + it("holds with a machine-readable reason for invalid thresholds", () => { + const decision = evaluateOraclePolicyRollout({ + ...baseInput, + minVerifiedSamples: 0, + maxSuccessRateRegression: -0.01, + maxCostRatio: 0, + }); + + expect(decision).toMatchObject({ status: "hold", sufficient: false }); + expect(decision.reasons).toEqual(["invalid_thresholds"]); + }); + + it("holds until both arms have enough verified samples", () => { + const decision = evaluateOraclePolicyRollout({ + ...baseInput, + treatment: { ...baseInput.treatment, verifiedSamples: 19, successes: 15 }, + }); + + expect(decision).toMatchObject({ status: "hold", sufficient: false }); + expect(decision.reasons).toContain("insufficient_verified_samples"); + }); + + it("holds when aggregate policy versions do not match the experiment", () => { + const decision = evaluateOraclePolicyRollout({ + ...baseInput, + treatment: { ...baseInput.treatment, policyVersion: "oracle-v3" }, + }); + + expect(decision).toMatchObject({ status: "hold", sufficient: false }); + expect(decision.reasons).toContain("mixed_policy_versions"); + }); + + it("rolls back on any verified safety violation", () => { + const decision = evaluateOraclePolicyRollout({ + ...baseInput, + treatment: { ...baseInput.treatment, safetyViolations: 1 }, + }); + + expect(decision.status).toBe("rollback"); + expect(decision.reasons).toContain("treatment_safety_violation"); + }); + + it("holds when treatment success regresses past the guardrail", () => { + const decision = evaluateOraclePolicyRollout({ + ...baseInput, + treatment: { ...baseInput.treatment, successes: 74 }, + }); + + expect(decision.status).toBe("hold"); + expect(decision.reasons).toContain("success_rate_regression"); + }); + + it("holds when cost or latency exceeds its configured ratio", () => { + const cost = evaluateOraclePolicyRollout({ + ...baseInput, + maxCostRatio: 1.1, + treatment: { ...baseInput.treatment, averageCostUsd: 1.11 }, + }); + const latency = evaluateOraclePolicyRollout({ + ...baseInput, + maxLatencyRatio: 1.1, + treatment: { ...baseInput.treatment, averageLatencyMs: 1_101 }, + }); + + expect(cost.reasons).toContain("cost_ratio_exceeded"); + expect(latency.reasons).toContain("latency_ratio_exceeded"); + }); + + it("promotes only when every verified outcome gate passes", () => { + const decision = evaluateOraclePolicyRollout({ + ...baseInput, + maxCostRatio: 1.1, + maxLatencyRatio: 1.1, + }); + + expect(decision).toMatchObject({ status: "promote", sufficient: true }); + expect(decision.reasons).toEqual(["all_rollout_gates_passed"]); + expect(decision.metrics).toMatchObject({ + controlSuccessRate: 0.8, + treatmentSuccessRate: 0.82, + costRatio: 1.05, + latencyRatio: 1.05, + }); + }); +}); diff --git a/test/agent/plugin-agent-loader.test.ts b/test/agent/plugin-agent-loader.test.ts new file mode 100644 index 000000000..b975b709c --- /dev/null +++ b/test/agent/plugin-agent-loader.test.ts @@ -0,0 +1,73 @@ +import { + mkdirSync, + mkdtempSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { loadPluginAgentMetadataFromDirectories } from "../../src/agent/plugin-agent-loader.js"; + +describe("plugin agent metadata loader", () => { + const roots: string[] = []; + + afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }); + } + }); + + function agent(key: string, entry = "index.js"): string { + const root = mkdtempSync(join(tmpdir(), "maestro-plugin-agent-")); + roots.push(root); + const directory = join(root, key); + mkdirSync(directory); + writeFileSync( + join(directory, "agent.json"), + JSON.stringify({ key, label: "Focused reviewer", entry }), + ); + writeFileSync(join(directory, "index.js"), "export default {};"); + return directory; + } + + it("loads immutable static metadata with resolved entries", () => { + const directory = agent("focused-reviewer"); + const result = loadPluginAgentMetadataFromDirectories({ + project: [directory], + }); + + expect(result.errors).toEqual([]); + expect(result.metadata).toEqual([ + expect.objectContaining({ + key: "focused-reviewer", + label: "Focused reviewer", + scope: "project", + entry: realpathSync(join(directory, "index.js")), + }), + ]); + expect(Object.isFrozen(result.metadata[0])).toBe(true); + }); + + it("rejects duplicate keys and entries that escape through symlinks", () => { + const first = agent("reviewer"); + const duplicate = agent("reviewer"); + const escaping = agent("escaping", "outside.js"); + const outside = join(roots.at(-1)!, "outside-target.js"); + writeFileSync(outside, "export default {};"); + symlinkSync(outside, join(escaping, "outside.js")); + + const result = loadPluginAgentMetadataFromDirectories({ + user: [first], + project: [duplicate, escaping], + }); + + expect(result.metadata).toHaveLength(1); + expect(result.errors).toEqual([ + expect.stringContaining("Duplicate plugin agent key: reviewer"), + expect.stringContaining("must remain inside"), + ]); + }); +}); diff --git a/test/agent/plugin-agent-registry.test.ts b/test/agent/plugin-agent-registry.test.ts new file mode 100644 index 000000000..592253eef --- /dev/null +++ b/test/agent/plugin-agent-registry.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, it } from "vitest"; +import { createPluginAgentApi } from "../../src/agent/plugin-agent-api.js"; + +const policy = { + allowedModels: ["anthropic/claude-sonnet-4-6"], + allowedTools: ["read", "search", "todo"], + maxBudgets: { maxTurns: 20, maxToolCalls: 40, maxCostUsd: 10 }, + approvalMode: "prompt" as const, + sandboxMode: "workspace-write" as const, +}; + +const baseConfig = { + key: "focused-reviewer", + label: "Focused reviewer", + description: "Reviews a bounded change", + systemPrompt: "Review the requested change and report actionable findings.", + model: "anthropic/claude-sonnet-4-6", + tools: ["read", "search"] as const, + budgets: { maxTurns: 10, maxToolCalls: 20, maxCostUsd: 5 }, + approvalMode: "fail" as const, + sandboxMode: "read-only" as const, +}; + +function api() { + return createPluginAgentApi({ + policy, + metadata: [ + { + key: "focused-reviewer", + label: "Focused reviewer", + entry: "agents/focused-reviewer.js", + }, + ], + }); +} + +describe("governed plugin agent registry", () => { + it("creates immutable handles within host policy", () => { + const handle = api().createAgent(baseConfig); + + expect(handle).toMatchObject({ + key: "focused-reviewer", + model: "anthropic/claude-sonnet-4-6", + tools: ["read", "search"], + budgets: { maxTurns: 10, maxToolCalls: 20, maxCostUsd: 5 }, + }); + expect(Object.isFrozen(handle)).toBe(true); + expect(Object.isFrozen(handle.tools)).toBe(true); + }); + + it("registers a validated primary mode", () => { + const registry = api(); + const handle = registry.createAgent(baseConfig); + + registry.registerAgentMode({ + key: "focused-reviewer", + label: "Focused reviewer", + agent: handle, + primary: true, + }); + + expect(registry.getAgentMode("focused-reviewer")).toMatchObject({ + primary: true, + agent: handle, + }); + }); + + it("rejects duplicate mode keys atomically", () => { + const registry = api(); + const handle = registry.createAgent(baseConfig); + const registration = { + key: "focused-reviewer", + label: "Focused reviewer", + agent: handle, + primary: false, + }; + registry.registerAgentMode(registration); + + expect(() => registry.registerAgentMode(registration)).toThrow( + /duplicate/i, + ); + expect(registry.listAgentModes()).toHaveLength(1); + }); + + it("rejects unknown tools and disallowed models", () => { + expect(() => api().createAgent({ ...baseConfig, tools: ["bash"] })).toThrow( + /tool/i, + ); + expect(() => + api().createAgent({ ...baseConfig, model: "openai/gpt-5.5" }), + ).toThrow(/model/i); + }); + + it("resolves tools all to the host allowlist", () => { + expect(api().createAgent({ ...baseConfig, tools: "all" }).tools).toEqual([ + "read", + "search", + "todo", + ]); + }); + + it("rejects unbounded or excessive budgets", () => { + expect(() => + api().createAgent({ + ...baseConfig, + budgets: { ...baseConfig.budgets, maxTurns: 0 }, + }), + ).toThrow(/budget/i); + expect(() => + api().createAgent({ + ...baseConfig, + budgets: { ...baseConfig.budgets, maxToolCalls: 41 }, + }), + ).toThrow(/budget/i); + expect(() => + api().createAgent({ + ...baseConfig, + budgets: { ...baseConfig.budgets, maxCostUsd: 11 }, + }), + ).toThrow(/budget/i); + }); + + it("rejects metadata mismatches", () => { + const registry = api(); + const handle = registry.createAgent(baseConfig); + + expect(() => + registry.registerAgentMode({ + key: "focused-reviewer", + label: "Different label", + agent: handle, + primary: true, + }), + ).toThrow(/metadata/i); + expect(registry.listAgentModes()).toHaveLength(0); + }); + + it("rejects approval and sandbox permission escalation", () => { + expect(() => + api().createAgent({ ...baseConfig, approvalMode: "auto" }), + ).toThrow(/approval/i); + expect(() => + api().createAgent({ ...baseConfig, sandboxMode: "danger-full-access" }), + ).toThrow(/sandbox/i); + }); +}); diff --git a/test/agent/routing-receipt.test.ts b/test/agent/routing-receipt.test.ts new file mode 100644 index 000000000..9dc1d340f --- /dev/null +++ b/test/agent/routing-receipt.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from "vitest"; +import { resolveAgentProfile } from "../../src/agent/profiles.js"; +import { + createRoutingReceipt, + resolveAgentProfileSelection, +} from "../../src/agent/routing-receipt.js"; +import type { RoutingDecision } from "../../src/services/intelligent-router/types.js"; + +function decision(): RoutingDecision { + return { + decisionId: "decision-42", + taskType: "coding", + strategy: "quality", + selectedModel: { provider: "anthropic", model: "claude-opus-4-6" }, + selectedProfile: resolveAgentProfile("high", "anthropic"), + fallbackProfiles: [resolveAgentProfile("medium", "anthropic")], + fallbackChain: [{ provider: "openai", model: "gpt-5.4" }], + scores: [], + overrideApplied: false, + reason: "highest_score", + createdAt: "2026-07-14T12:00:00.000Z", + oracleConsultation: { + policyVersion: "evalops.maestro.oracle-consultation.v1", + evalSuite: "oracle-consultation-policy-v1", + mode: "recommended", + reasons: ["high_profile"], + }, + }; +} + +describe("resolveAgentProfileSelection", () => { + it("prefers an explicit request over the session pin and compatibility default", () => { + expect( + resolveAgentProfileSelection({ + requestedProfile: "high", + sessionPin: { profile: "medium", updatedAt: "2026-07-14T11:00:00Z" }, + compatibilityProfile: "low", + }), + ).toEqual({ requestedProfile: "high", source: "request" }); + }); + + it("uses the session pin before the compatibility default", () => { + expect( + resolveAgentProfileSelection({ + sessionPin: { profile: "medium", updatedAt: "2026-07-14T11:00:00Z" }, + compatibilityProfile: "low", + }), + ).toEqual({ requestedProfile: "medium", source: "session" }); + }); + + it("falls back to the compatibility profile", () => { + expect( + resolveAgentProfileSelection({ compatibilityProfile: "low" }), + ).toEqual({ requestedProfile: "low", source: "compatibility_default" }); + }); +}); + +describe("createRoutingReceipt", () => { + it("projects an immutable routing receipt with Oracle, fallback, and experiment detail", () => { + const receipt = createRoutingReceipt(decision(), { + requestedProfile: "high", + source: "session", + fallbackReason: "primary_unavailable", + experiment: { + experimentId: "oracle-policy-2026-07", + arm: "treatment", + policyVersion: "oracle-v2", + }, + }); + + expect(receipt).toMatchObject({ + decisionId: "decision-42", + requestedProfile: "high", + source: "session", + resolvedProfileId: "high-v1", + resolvedProfileVersion: 1, + provider: "anthropic", + model: "claude-opus-4-6", + reasoningEffort: "xhigh", + createdAt: "2026-07-14T12:00:00.000Z", + oracle: { + policyVersion: "evalops.maestro.oracle-consultation.v1", + mode: "recommended", + reasons: ["high_profile"], + }, + fallback: { reason: "primary_unavailable" }, + experiment: { + experimentId: "oracle-policy-2026-07", + arm: "treatment", + policyVersion: "oracle-v2", + }, + }); + expect(Object.isFrozen(receipt)).toBe(true); + expect(Object.isFrozen(receipt.oracle)).toBe(true); + expect(Object.isFrozen(receipt.experiment)).toBe(true); + }); +}); diff --git a/test/app-server/host-control-api.test.ts b/test/app-server/host-control-api.test.ts index fcfa222ce..cdcbc15ea 100644 --- a/test/app-server/host-control-api.test.ts +++ b/test/app-server/host-control-api.test.ts @@ -350,10 +350,14 @@ describe("Maestro app-server host-control API", () => { }); expect(writeResponse.result).toEqual({}); - const change = await waitForNotification( - notifications, - (notification) => notification.method === "fs/changed", - ); + const change = await waitForNotification(notifications, (notification) => { + if (notification.method !== "fs/changed") return false; + const params = notification.params as { changedPaths?: unknown }; + return ( + Array.isArray(params.changedPaths) && + params.changedPaths.includes(filePath) + ); + }); expect(change.params).toMatchObject({ watchId: "source", changedPaths: [filePath], diff --git a/test/app-server/plugin-bundle-api.test.ts b/test/app-server/plugin-bundle-api.test.ts index 8b1c446c8..32ef9b142 100644 --- a/test/app-server/plugin-bundle-api.test.ts +++ b/test/app-server/plugin-bundle-api.test.ts @@ -26,7 +26,9 @@ function writeTrustedGlobalConfig(projectRoot: string): void { function writePluginBundle(root: string): string { const packageDir = join(root, "vendor", "review-bundle"); const skillDir = join(packageDir, "skills", "reviewing"); + const agentDir = join(packageDir, "agents", "reviewer"); mkdirSync(skillDir, { recursive: true }); + mkdirSync(agentDir, { recursive: true }); writeFileSync( join(packageDir, "package.json"), JSON.stringify( @@ -34,13 +36,18 @@ function writePluginBundle(root: string): string { name: "@test/maestro-review-bundle", version: "1.0.0", keywords: ["maestro-package"], - maestro: { skills: ["./skills"] }, + maestro: { agents: ["./agents"], skills: ["./skills"] }, }, null, 2, ), "utf8", ); + writeFileSync( + join(agentDir, "agent.json"), + JSON.stringify({ key: "reviewer", label: "Reviewer", entry: "./index.js" }), + "utf8", + ); writeFileSync( join(skillDir, "SKILL.md"), "---\nname: reviewing\ndescription: Review imported through plugin bundle lifecycle.\n---\n\n# Reviewing\n", @@ -184,6 +191,9 @@ describe("Maestro app-server plugin bundle lifecycle API", () => { expect(listed.result?.resources.skills.project).toEqual( expect.arrayContaining([join(packageDir, "skills", "reviewing")]), ); + expect(listed.result?.resources.agents?.project).toEqual([ + join(packageDir, "agents", "reviewer"), + ]); expect(loadSkills(projectRoot, { includeSystem: false }).skills).toEqual( expect.arrayContaining([ expect.objectContaining({ diff --git a/test/cli/painter.test.ts b/test/cli/painter.test.ts new file mode 100644 index 000000000..d9b5fcf12 --- /dev/null +++ b/test/cli/painter.test.ts @@ -0,0 +1,118 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + formatPainterHelp, + handlePainterCommand, + resolveInlineDisplay, +} from "../../src/cli/commands/painter.js"; + +function makeSink(isTTY = true): { + stream: NodeJS.WritableStream; + chunks: string[]; +} { + const chunks: string[] = []; + const stream = { + write: (c: unknown) => { + chunks.push(String(c)); + return true; + }, + isTTY, + }; + return { stream: stream as unknown as NodeJS.WritableStream, chunks }; +} + +describe("painter cli: formatPainterHelp", () => { + it("documents the show subcommand", () => { + const help = formatPainterHelp(); + expect(help).toContain("maestro painter show"); + expect(help).toMatch(/iTerm2|WezTerm|kitty/); + }); +}); + +describe("painter cli: resolveInlineDisplay", () => { + const png = Buffer.from([0x89, 0x50, 0x4e, 0x47]); + + it("produces an escape for a supported terminal", () => { + const r = resolveInlineDisplay(png, { support: "iterm", isTTY: true }); + expect(r.ok).toBe(true); + expect(r.escape?.startsWith("\x1b]1337;File=")).toBe(true); + }); + + it("rejects when the terminal is unsupported", () => { + const r = resolveInlineDisplay(png, { support: "none", isTTY: true }); + expect(r.ok).toBe(false); + expect(r.reason).toMatch(/not supported/); + expect(r.reason).toContain("none"); + }); + + it("rejects when stdout is not a TTY", () => { + const r = resolveInlineDisplay(png, { support: "iterm", isTTY: false }); + expect(r.ok).toBe(false); + expect(r.reason).toMatch(/not a TTY/); + }); + + it("rejects sixel (rasterization not implemented)", () => { + const r = resolveInlineDisplay(png, { support: "sixel", isTTY: true }); + expect(r.ok).toBe(false); + }); +}); + +describe("painter cli: handlePainterCommand", () => { + let savedExitCode: number | undefined; + const errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + + beforeEach(() => { + savedExitCode = process.exitCode; + process.exitCode = undefined; + }); + + afterEach(() => { + process.exitCode = savedExitCode; + }); + + it("prints help when no subcommand is given", async () => { + const { stream, chunks } = makeSink(); + await handlePainterCommand(undefined, [], stream); + expect(chunks.join("")).toContain("maestro painter show"); + }); + + it("errors on `show` with no path", async () => { + const { stream } = makeSink(); + await handlePainterCommand("show", [], stream); + expect(process.exitCode).toBe(1); + expect(errSpy).toHaveBeenCalledWith(expect.stringMatching(/image path/)); + }); + + it("errors when the path cannot be read", async () => { + const { stream } = makeSink(); + await handlePainterCommand("show", ["/nonexistent/x.png"], stream); + expect(process.exitCode).toBe(1); + expect(errSpy).toHaveBeenCalledWith( + expect.stringMatching(/Could not read image/), + ); + }); + + it("writes the inline escape to the sink for a supported terminal", async () => { + const dir = mkdtempSync(join(tmpdir(), "painter-cli-")); + try { + const path = join(dir, "img.png"); + writeFileSync(path, Buffer.from([0x89, 0x50, 0x4e, 0x47])); + const { stream, chunks } = makeSink(true); + // Force iTerm2 detection for the duration of the call. + const prev = process.env.TERM_PROGRAM; + process.env.TERM_PROGRAM = "iTerm.app"; + try { + await handlePainterCommand("show", [path], stream); + } finally { + if (prev === undefined) delete process.env.TERM_PROGRAM; + else process.env.TERM_PROGRAM = prev; + } + expect(process.exitCode).toBeUndefined(); + expect(chunks.join("").startsWith("\x1b]1337;File=")).toBe(true); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/packages/ai-tsconfig-boundary.test.ts b/test/packages/ai-tsconfig-boundary.test.ts new file mode 100644 index 000000000..ba9eec87c --- /dev/null +++ b/test/packages/ai-tsconfig-boundary.test.ts @@ -0,0 +1,14 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +describe("@evalops/ai TypeScript boundary", () => { + it("includes painter image-provider dependencies", () => { + const config = JSON.parse( + readFileSync("packages/ai/tsconfig.build.json", "utf8"), + ) as { include?: string[] }; + + expect(config.include).toContain( + "../../src/services/image-providers/**/*.ts", + ); + }); +}); diff --git a/test/packages/maestro-packages.test.ts b/test/packages/maestro-packages.test.ts index 736d8871f..61cb43320 100644 --- a/test/packages/maestro-packages.test.ts +++ b/test/packages/maestro-packages.test.ts @@ -630,6 +630,28 @@ describe("Maestro Packages", () => { }); describe("Resource Loading", () => { + it("should discover and filter custom agents from packages", async () => { + const pkgDir = join(testDir, "agent-package"); + mkdirSync(join(pkgDir, "agents", "reviewer"), { recursive: true }); + mkdirSync(join(pkgDir, "agents", "planner"), { recursive: true }); + writeFileSync( + join(pkgDir, "package.json"), + JSON.stringify({ + name: "@test/agents", + keywords: ["maestro-package"], + maestro: { agents: ["./agents"] }, + }), + ); + + const pkg = await loadPackage({ + source: `local:${pkgDir}`, + agents: ["reviewer"], + }); + const resources = loadPackageResources(pkg); + + expect(resources.agents).toEqual([join(pkgDir, "agents", "reviewer")]); + }); + it("should load extensions from package", async () => { const pkgDir = join(testDir, "ext-package"); mkdirSync(join(pkgDir, "extensions", "test-ext"), { recursive: true }); diff --git a/test/services/image-providers/cost.test.ts b/test/services/image-providers/cost.test.ts new file mode 100644 index 000000000..256506beb --- /dev/null +++ b/test/services/image-providers/cost.test.ts @@ -0,0 +1,250 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + PainterBudget, + estimateImageCost, + getPainterBudget, + resetPainterBudgetSingleton, +} from "../../../src/services/image-providers/cost.js"; + +const COST_ENV = [ + "MAESTRO_PAINTER_MAX_COST_CENTS", + "MAESTRO_PAINTER_PRICE_TABLE", +]; + +describe("cost: estimateImageCost", () => { + const saved: Record = {}; + + beforeEach(() => { + for (const k of COST_ENV) { + saved[k] = process.env[k]; + delete process.env[k]; + } + }); + + afterEach(() => { + for (const k of COST_ENV) { + if (saved[k] === undefined) delete process.env[k]; + else process.env[k] = saved[k]; + } + }); + + it("returns undefined when no price is configured", () => { + expect( + estimateImageCost({ + model: "gpt-image-2", + size: "1024x1024", + quality: "high", + }), + ).toBeUndefined(); + }); + + it("resolves an exact model/size/quality key", () => { + process.env.MAESTRO_PAINTER_PRICE_TABLE = JSON.stringify({ + "gpt-image-2|1024x1024|high": 8, + }); + expect( + estimateImageCost({ + model: "gpt-image-2", + size: "1024x1024", + quality: "high", + }), + ).toBe(8); + }); + + it("multiplies by n", () => { + process.env.MAESTRO_PAINTER_PRICE_TABLE = JSON.stringify({ + "gpt-image-2|1024x1024|high": 8, + }); + expect( + estimateImageCost({ + model: "gpt-image-2", + size: "1024x1024", + quality: "high", + n: 3, + }), + ).toBe(24); + }); + + it("falls back to size wildcard, then quality wildcard, then model wildcard", () => { + process.env.MAESTRO_PAINTER_PRICE_TABLE = JSON.stringify({ + "gpt-image-2|*|high": 8, + "gpt-image-2|*|*": 4, + }); + expect( + estimateImageCost({ + model: "gpt-image-2", + size: "1792x1024", + quality: "high", + }), + ).toBe(8); + expect( + estimateImageCost({ + model: "gpt-image-2", + size: "1792x1024", + quality: "low", + }), + ).toBe(4); + }); + + it("ignores non-numeric entries in the table", () => { + process.env.MAESTRO_PAINTER_PRICE_TABLE = JSON.stringify({ + "gpt-image-2|1024x1024|high": "free", + }); + expect( + estimateImageCost({ + model: "gpt-image-2", + size: "1024x1024", + quality: "high", + }), + ).toBeUndefined(); + }); + + it("ignores malformed JSON", () => { + process.env.MAESTRO_PAINTER_PRICE_TABLE = "{not json"; + expect( + estimateImageCost({ + model: "gpt-image-2", + size: "1024x1024", + quality: "high", + }), + ).toBeUndefined(); + }); +}); + +describe("cost: PainterBudget", () => { + beforeEach(() => { + for (const k of COST_ENV) delete process.env[k]; + resetPainterBudgetSingleton(); + }); + + afterEach(() => { + for (const k of COST_ENV) delete process.env[k]; + resetPainterBudgetSingleton(); + }); + + it("does not enforce when no ceiling is configured", () => { + process.env.MAESTRO_PAINTER_PRICE_TABLE = JSON.stringify({ + "gpt-image-2|1024x1024|high": 8, + }); + const budget = new PainterBudget(); + const d = budget.checkAndReserve({ + model: "gpt-image-2", + size: "1024x1024", + quality: "high", + }); + expect(d.ok).toBe(true); + expect(d.enforced).toBe(false); + }); + + it("fails open when a ceiling is set but prices are unknown", () => { + process.env.MAESTRO_PAINTER_MAX_COST_CENTS = "100"; + const budget = new PainterBudget(); + const d = budget.checkAndReserve({ + model: "gpt-image-2", + size: "1024x1024", + quality: "high", + }); + expect(d.ok).toBe(true); + expect(d.enforced).toBe(false); + expect(d.reason).toMatch(/no price configured/); + }); + + it("reserves and accumulates when under the ceiling", () => { + process.env.MAESTRO_PAINTER_MAX_COST_CENTS = "100"; + process.env.MAESTRO_PAINTER_PRICE_TABLE = JSON.stringify({ + "gpt-image-2|1024x1024|high": 30, + }); + const budget = new PainterBudget(); + const first = budget.checkAndReserve({ + model: "gpt-image-2", + size: "1024x1024", + quality: "high", + n: 2, + }); + expect(first.ok).toBe(true); + expect(first.enforced).toBe(true); + expect(first.estimatedCents).toBe(60); + expect(first.cumulativeCents).toBe(60); + + const second = budget.checkAndReserve({ + model: "gpt-image-2", + size: "1024x1024", + quality: "high", + }); + expect(second.ok).toBe(true); + expect(second.cumulativeCents).toBe(90); + }); + + it("rejects when the next call would exceed the ceiling", () => { + process.env.MAESTRO_PAINTER_MAX_COST_CENTS = "50"; + process.env.MAESTRO_PAINTER_PRICE_TABLE = JSON.stringify({ + "gpt-image-2|1024x1024|high": 30, + }); + const budget = new PainterBudget(); + expect( + budget.checkAndReserve({ + model: "gpt-image-2", + size: "1024x1024", + quality: "high", + }).ok, + ).toBe(true); // 30 reserved + const blocked = budget.checkAndReserve({ + model: "gpt-image-2", + size: "1024x1024", + quality: "high", + }); + expect(blocked.ok).toBe(false); + expect(blocked.enforced).toBe(true); + expect(blocked.reason).toMatch(/exceeded/); + expect(blocked.cumulativeCents).toBe(30); // not reserved + }); + + it("reset() clears accumulated spend", () => { + process.env.MAESTRO_PAINTER_MAX_COST_CENTS = "100"; + process.env.MAESTRO_PAINTER_PRICE_TABLE = JSON.stringify({ + "gpt-image-2|1024x1024|high": 30, + }); + const budget = new PainterBudget(); + budget.checkAndReserve({ + model: "gpt-image-2", + size: "1024x1024", + quality: "high", + }); + expect(budget.cumulativeCents).toBe(30); + budget.reset(); + expect(budget.cumulativeCents).toBe(0); + }); + + it("release() refunds reserved spend and clamps at zero", () => { + process.env.MAESTRO_PAINTER_MAX_COST_CENTS = "100"; + process.env.MAESTRO_PAINTER_PRICE_TABLE = JSON.stringify({ + "gpt-image-2|1024x1024|high": 30, + }); + const budget = new PainterBudget(); + budget.checkAndReserve({ + model: "gpt-image-2", + size: "1024x1024", + quality: "high", + }); + expect(budget.cumulativeCents).toBe(30); + budget.release(30); // refund a failed call + expect(budget.cumulativeCents).toBe(0); + budget.release(50); // cannot go negative / raise the ceiling + expect(budget.cumulativeCents).toBe(0); + }); + + it("getPainterBudget returns a process singleton", () => { + const a = getPainterBudget(); + const b = getPainterBudget(); + expect(a).toBe(b); + resetPainterBudgetSingleton(); + const c = getPainterBudget(); + expect(c).not.toBe(a); + }); + + it("ignores a non-numeric ceiling value", () => { + process.env.MAESTRO_PAINTER_MAX_COST_CENTS = "not-a-number"; + const budget = new PainterBudget(); + expect(budget.ceilingCents).toBeUndefined(); + }); +}); diff --git a/test/services/image-providers/flux.test.ts b/test/services/image-providers/flux.test.ts new file mode 100644 index 000000000..a81e93ed1 --- /dev/null +++ b/test/services/image-providers/flux.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, it, vi } from "vitest"; +import { + FluxImageProvider, + type FluxTransport, + mapSizeToFlux, +} from "../../../src/services/image-providers/flux.js"; + +const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47]); + +function makeTransport(responses: { + post?: { images: { url?: string; content_type?: string }[] }; + get?: Buffer; +}): FluxTransport & { + postSpy: ReturnType; + getSpy: ReturnType; +} { + const postSpy = vi + .fn() + .mockResolvedValue((responses.post ?? { images: [] }) as never); + const getSpy = vi + .fn() + .mockResolvedValue((responses.get ?? PNG) as never); + return { post: postSpy, get: getSpy, postSpy, getSpy }; +} + +describe("flux: mapSizeToFlux", () => { + it("maps our sizes to fal's enum", () => { + expect(mapSizeToFlux("1024x1024")).toBe("square"); + expect(mapSizeToFlux("1536x1024")).toBe("landscape_4_3"); + expect(mapSizeToFlux("1792x1024")).toBe("landscape_4_3"); + expect(mapSizeToFlux("1024x1536")).toBe("portrait_4_3"); + expect(mapSizeToFlux("1024x1792")).toBe("portrait_4_3"); + expect(mapSizeToFlux("auto")).toBe("square_hd"); + expect(mapSizeToFlux(undefined)).toBe("square_hd"); + }); +}); + +describe("flux: generate", () => { + it("posts to the model endpoint and persists fetched bytes via writeImage", async () => { + const writeImage = vi + .fn<(b: Buffer, ext: string) => Promise<{ path: string }>>() + .mockResolvedValue({ path: "/tmp/out.png" }); + const transport = makeTransport({ + post: { + images: [{ url: "https://cdn/x.png", content_type: "image/png" }], + }, + get: PNG, + }); + const provider = new FluxImageProvider({ + apiKey: "fal-key", + writeImage, + transport, + }); + + const result = await provider.generate({ prompt: "an icon", n: 2 }); + + // Request shape: fal endpoint, image_size mapping, num_images, png output. + expect(transport.postSpy).toHaveBeenCalledOnce(); + const [url, body] = transport.postSpy.mock.calls[0]!; + expect(url).toBe("https://fal.run/fal-ai/flux/schnell"); + expect(body).toMatchObject({ + prompt: "an icon", + image_size: "square_hd", + num_images: 2, + output_format: "png", + }); + + // Each returned URL is fetched once and persisted once. + expect(transport.getSpy).toHaveBeenCalledTimes(1); + expect(writeImage).toHaveBeenCalledTimes(1); + expect(writeImage.mock.calls[0]![1]).toBe("png"); + + expect(result.provider).toBe("flux"); + expect(result.model).toBe("fal-ai/flux/schnell"); + expect(result.images[0]?.path).toBe("/tmp/out.png"); + expect(result.images[0]?.mimeType).toBe("image/png"); + }); + + it("throws when the API returns no images", async () => { + const provider = new FluxImageProvider({ + apiKey: "fal-key", + writeImage: vi.fn(), + transport: makeTransport({ post: { images: [] } }), + }); + await expect(provider.generate({ prompt: "x" })).rejects.toThrow( + /no images/i, + ); + }); + + it("throws when images lack downloadable URLs", async () => { + const provider = new FluxImageProvider({ + apiKey: "fal-key", + writeImage: vi.fn(), + transport: makeTransport({ post: { images: [{ url: undefined }] } }), + }); + await expect(provider.generate({ prompt: "x" })).rejects.toThrow( + /no downloadable/i, + ); + }); + + it("honors a custom model id", async () => { + const transport = makeTransport({ + post: { images: [{ url: "https://cdn/y.png" }] }, + }); + const provider = new FluxImageProvider({ + apiKey: "fal-key", + model: "fal-ai/flux/dev", + writeImage: vi.fn().mockResolvedValue({ path: "/tmp/y.png" }), + transport, + }); + await provider.generate({ prompt: "x" }); + const [url] = transport.postSpy.mock.calls[0]!; + expect(url).toBe("https://fal.run/fal-ai/flux/dev"); + }); +}); + +describe("flux: edit + supports", () => { + it("declares generate-only support", () => { + const provider = new FluxImageProvider({ + apiKey: "fal-key", + writeImage: vi.fn(), + transport: makeTransport({}), + }); + expect(provider.supports).toEqual({ + generate: true, + edit: false, + mask: false, + }); + }); + + it("rejects edit calls explicitly", async () => { + const provider = new FluxImageProvider({ + apiKey: "fal-key", + writeImage: vi.fn(), + transport: makeTransport({}), + }); + await expect( + provider.edit({ prompt: "x", images: ["a.png"] }), + ).rejects.toThrow(/does not support editing/); + }); +}); diff --git a/test/services/image-providers/masks.test.ts b/test/services/image-providers/masks.test.ts new file mode 100644 index 000000000..f08d3e929 --- /dev/null +++ b/test/services/image-providers/masks.test.ts @@ -0,0 +1,138 @@ +import { inflateSync } from "node:zlib"; +import { describe, expect, it } from "vitest"; +import { + type ImageDimensions, + type MaskRegion, + clipRegion, + crc32, + encodeMaskPng, + readPngDimensions, +} from "../../../src/services/image-providers/masks.js"; + +function findChunk(png: Buffer, type: string): Buffer | null { + let offset = 8; // skip signature + while (offset + 8 <= png.length) { + const len = png.readUInt32BE(offset); + const chunkType = png.toString("ascii", offset + 4, offset + 8); + const dataStart = offset + 8; + if (chunkType === type) return png.subarray(dataStart, dataStart + len); + offset = dataStart + len + 4; // skip data + crc + } + return null; +} + +/** Inflate the (single) IDAT and return raw RGBA scanlines, filter byte stripped. */ +function rawPixels(png: Buffer): { + width: number; + height: number; + data: Buffer; +} { + const ihdr = findChunk(png, "IHDR")!; + const width = ihdr.readUInt32BE(0); + const height = ihdr.readUInt32BE(4); + const idat = findChunk(png, "IDAT")!; + const inflated = Buffer.from(inflateSync(idat)); + // strip per-row filter byte + const rowLen = width * 4; + const data = Buffer.alloc(rowLen * height); + for (let y = 0; y < height; y++) { + inflated + .subarray(y * (rowLen + 1) + 1, y * (rowLen + 1) + 1 + rowLen) + .copy(data, y * rowLen); + } + return { width, height, data }; +} + +describe("masks: crc32", () => { + it("matches the canonical CRC32 check value", () => { + // RFC 1952 / PKZIP reference: crc32("123456789") == 0xCBF43926 + expect(crc32(Buffer.from("123456789", "ascii"))).toBe(0xcbf43926); + }); + + it("is stable for empty input", () => { + expect(crc32(Buffer.alloc(0))).toBe(0x00000000); + }); +}); + +describe("masks: clipRegion", () => { + const canvas: ImageDimensions = { width: 100, height: 100 }; + + it("clips a region that overflows the right/bottom edges", () => { + const r = clipRegion(canvas, { x: 90, y: 90, width: 50, height: 50 }); + expect(r).toEqual({ x: 90, y: 90, width: 10, height: 10 }); + }); + + it("clamps negative origin to zero", () => { + const r = clipRegion(canvas, { x: -10, y: -10, width: 20, height: 20 }); + expect(r).toEqual({ x: 0, y: 0, width: 10, height: 10 }); + }); + + it("returns empty width when the region is fully outside", () => { + const r = clipRegion(canvas, { x: 200, y: 0, width: 10, height: 10 }); + expect(r.width).toBe(0); + }); +}); + +describe("masks: encodeMaskPng", () => { + const canvas: ImageDimensions = { width: 10, height: 6 }; + const region: MaskRegion = { x: 2, y: 1, width: 4, height: 2 }; + + it("emits a valid PNG signature and IHDR matching the canvas", () => { + const png = encodeMaskPng(canvas, region); + expect(png.subarray(0, 8).equals(png.subarray(0, 8))).toBe(true); + expect(readPngDimensions(png)).toEqual(canvas); + }); + + it("marks pixels inside the region as transparent (alpha 0) and outside as opaque (255)", () => { + const png = encodeMaskPng(canvas, region); + const { width, data } = rawPixels(png); + const alpha = (x: number, y: number) => data[(y * width + x) * 4 + 3]; + + // inside the region + expect(alpha(2, 1)).toBe(0); + expect(alpha(5, 2)).toBe(0); + // outside the region (corners + adjacent) + expect(alpha(0, 0)).toBe(255); + expect(alpha(9, 5)).toBe(255); + expect(alpha(6, 1)).toBe(255); // just right of region + expect(alpha(2, 3)).toBe(255); // just below region + }); + + it("clips a region that exceeds the canvas instead of failing", () => { + const png = encodeMaskPng(canvas, { x: 8, y: 5, width: 10, height: 10 }); + const { width, data } = rawPixels(png); + // clipped region is x in [8,10), y in [5,6): pixel (8,5) transparent + expect(data[(5 * width + 8) * 4 + 3]).toBe(0); + expect(data[(5 * width + 7) * 4 + 3]).toBe(255); + }); + + it("throws on non-positive canvas dimensions", () => { + expect(() => encodeMaskPng({ width: 0, height: 10 }, region)).toThrow( + /positive/, + ); + }); + + it("throws when the region is empty after clipping", () => { + expect(() => + encodeMaskPng(canvas, { x: 200, y: 200, width: 5, height: 5 }), + ).toThrow(/empty after clipping/); + }); +}); + +describe("masks: readPngDimensions", () => { + it("round-trips through encodeMaskPng", () => { + const png = encodeMaskPng( + { width: 64, height: 32 }, + { x: 0, y: 0, width: 8, height: 8 }, + ); + expect(readPngDimensions(png)).toEqual({ width: 64, height: 32 }); + }); + + it("returns null for non-PNG input", () => { + expect(readPngDimensions(Buffer.from("not a png"))).toBeNull(); + }); + + it("returns null for a truncated buffer", () => { + expect(readPngDimensions(Buffer.alloc(10))).toBeNull(); + }); +}); diff --git a/test/services/image-providers/openai.test.ts b/test/services/image-providers/openai.test.ts new file mode 100644 index 000000000..07f95e80a --- /dev/null +++ b/test/services/image-providers/openai.test.ts @@ -0,0 +1,145 @@ +import { rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type OpenAI from "openai"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { OpenAIImageProvider } from "../../../src/services/image-providers/openai.js"; + +/** + * Build a stub OpenAI client whose images.generate / images.edit return canned + * payloads. The provider only depends on `client.images.{generate,edit}`, so + * this is enough to exercise decode + write + mask handling without network. + */ +function makeStubClient(responses: { + generate?: unknown; + edit?: unknown; +}): OpenAI { + const generate = vi + .fn() + .mockResolvedValue(responses.generate as never); + const edit = vi + .fn() + .mockResolvedValue(responses.edit as never); + return { images: { generate, edit } } as unknown as OpenAI; +} + +const TINY_PNG_BASE64 = Buffer.from([0x89, 0x50, 0x4e, 0x47]).toString( + "base64", +); + +describe("OpenAIImageProvider", () => { + const tmpRoot = join( + tmpdir(), + `painter-provider-${process.pid}-${Date.now()}`, + ); + + beforeEach(async () => { + process.env.MAESTRO_PAINTER_OUTPUT_DIR = tmpRoot; + }); + + afterEach(async () => { + delete process.env.MAESTRO_PAINTER_OUTPUT_DIR; + await rm(tmpRoot, { recursive: true, force: true }); + }); + + describe("generate", () => { + it("decodes b64_json and persists via writeImage", async () => { + const writeImage = vi + .fn<(b: Buffer, ext: string) => Promise<{ path: string }>>() + .mockResolvedValue({ path: "/tmp/out.png" }); + const client = makeStubClient({ + generate: { data: [{ b64_json: TINY_PNG_BASE64 }] }, + }); + const provider = new OpenAIImageProvider({ + client, + writeImage, + model: "gpt-image-2", + }); + + const result = await provider.generate({ prompt: "an icon" }); + + expect(writeImage).toHaveBeenCalledTimes(1); + const [bytes, ext] = writeImage.mock.calls[0]!; + expect(ext).toBe("png"); + expect(bytes.length).toBe(4); + expect(result.images[0]?.path).toBe("/tmp/out.png"); + expect(result.provider).toBe("openai"); + expect(result.model).toBe("gpt-image-2"); + expect(client.images.generate).toHaveBeenCalledOnce(); + }); + + it("carries the revised prompt through when present", async () => { + const writeImage = vi.fn().mockResolvedValue({ path: "/tmp/out.png" }); + const client = makeStubClient({ + generate: { + data: [ + { b64_json: TINY_PNG_BASE64, revised_prompt: "a better icon" }, + ], + }, + }); + const provider = new OpenAIImageProvider({ client, writeImage }); + + const result = await provider.generate({ prompt: "icon" }); + + expect(result.images[0]?.revisedPrompt).toBe("a better icon"); + }); + + it("throws when the API returns no image data", async () => { + const writeImage = vi.fn(); + const client = makeStubClient({ generate: { data: [] } }); + const provider = new OpenAIImageProvider({ client, writeImage }); + + await expect(provider.generate({ prompt: "x" })).rejects.toThrow( + /no image data/i, + ); + expect(writeImage).not.toHaveBeenCalled(); + }); + }); + + describe("edit", () => { + it("requires at least one input image path", async () => { + const provider = new OpenAIImageProvider({ + client: makeStubClient({ edit: { data: [] } }), + writeImage: vi.fn(), + }); + await expect(provider.edit({ prompt: "x", images: [] })).rejects.toThrow( + /at least one input image/, + ); + }); + + it("passes image files and omits mask when no maskPath", async () => { + const writeImage = vi.fn().mockResolvedValue({ path: "/tmp/o.png" }); + const client = makeStubClient({ + edit: { data: [{ b64_json: TINY_PNG_BASE64 }] }, + }); + const provider = new OpenAIImageProvider({ client, writeImage }); + + // Use a real temp file so readFile succeeds; contents are irrelevant. + const inputPath = join(tmpRoot, "input.png"); + const { writeFile, mkdir } = await import("node:fs/promises"); + await mkdir(tmpRoot, { recursive: true }); + await writeFile(inputPath, Buffer.from([0x89, 0x50])); + + await provider.edit({ prompt: "redact", images: [inputPath] }); + + const params = (client.images.edit as ReturnType).mock + .calls[0]![0] as { image: unknown[]; mask?: unknown }; + expect(params.image).toHaveLength(1); + expect(params.mask).toBeUndefined(); + }); + }); + + describe("supports", () => { + it("declares generate, edit, and mask", () => { + const provider = new OpenAIImageProvider({ + client: makeStubClient({}), + writeImage: vi.fn(), + }); + expect(provider.supports).toEqual({ + generate: true, + edit: true, + mask: true, + }); + }); + }); +}); diff --git a/test/services/intelligent-router.test.ts b/test/services/intelligent-router.test.ts index ba75f4f08..bba298c70 100644 --- a/test/services/intelligent-router.test.ts +++ b/test/services/intelligent-router.test.ts @@ -8,6 +8,7 @@ import { setIntelligentRouterServiceForTest, } from "../../src/services/intelligent-router/index.js"; import { resolveIntelligentRouterTaskSummary } from "../../src/services/intelligent-router/recorder.js"; +import { resolveIntelligentRouterProfileHint } from "../../src/services/intelligent-router/recorder.js"; import type { RoutingModelCandidate } from "../../src/services/intelligent-router/types.js"; interface MockResponse { @@ -77,6 +78,15 @@ function createService(): IntelligentRouterService { } describe("intelligent router service", () => { + it("resolves request profile headers before body hints", () => { + const req = createRequest("POST", "/api/chat", undefined, { + "x-maestro-agent-profile": " ultra ", + }); + expect(resolveIntelligentRouterProfileHint(req, { profile: "rush" })).toBe( + "ultra", + ); + }); + it("derives policy signals from the latest user message", () => { expect( resolveIntelligentRouterTaskSummary({ diff --git a/test/session/session-manager.test.ts b/test/session/session-manager.test.ts index 084458f54..587623bfe 100644 --- a/test/session/session-manager.test.ts +++ b/test/session/session-manager.test.ts @@ -504,6 +504,41 @@ describe("SessionManager - Deferred Session Creation", () => { ).toEqual(unifiedContextManifest); }); + it("persists a pending agent profile pin when the session starts", () => { + const sessionManager = new SessionManager(false); + const pin = { + profile: "high", + updatedAt: "2026-07-14T12:00:00.000Z", + }; + + expect(sessionManager.updateAgentProfilePin(pin)).toBe(true); + sessionManager.startSession(createMockState()); + + expect(sessionManager.getHeader()?.agentProfilePin).toEqual(pin); + expect( + readSessionHeader(sessionManager.getSessionFile()).agentProfilePin, + ).toEqual(pin); + }); + + it("updates an existing session profile pin without rewriting messages", () => { + const sessionManager = new SessionManager(false); + sessionManager.startSession(createMockState()); + sessionManager.saveMessage(createUserMessage("Keep this message")); + const entriesBefore = sessionManager.getEntries(); + + expect( + sessionManager.updateAgentProfilePin({ + profile: "medium", + updatedAt: "2026-07-14T12:01:00.000Z", + }), + ).toBe(true); + + expect(sessionManager.getEntries()).toEqual(entriesBefore); + expect(sessionManager.getHeader()?.agentProfilePin?.profile).toBe( + "medium", + ); + }); + it("does not duplicate buffered entries when backfilling the unified context manifest", async () => { const sessionManager = new SessionManager(false); const promptContextManifest = createPromptContextManifest(); diff --git a/test/telemetry/oracle-policy.test.ts b/test/telemetry/oracle-policy.test.ts new file mode 100644 index 000000000..287bcc6a7 --- /dev/null +++ b/test/telemetry/oracle-policy.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("../../src/telemetry.js", () => ({ + recordTelemetry: vi.fn(() => Promise.resolve()), +})); + +import { recordTelemetry } from "../../src/telemetry.js"; +import { recordOraclePolicyExperimentAssignment } from "../../src/telemetry/oracle-policy.js"; + +describe("Oracle policy experiment telemetry", () => { + it("emits bounded assignment attributes", () => { + recordOraclePolicyExperimentAssignment({ + sessionId: "session-1", + assignment: { + experimentId: "oracle-july", + arm: "treatment", + policyVersion: "oracle-v2", + bucket: 0.25, + }, + }); + + expect(recordTelemetry).toHaveBeenCalledWith( + expect.objectContaining({ + type: "staged-rollout-surface", + event: "internal_gate_used", + metadata: { + sessionId: "session-1", + "oracle.experiment_id": "oracle-july", + "oracle.arm": "treatment", + "oracle.policy_version": "oracle-v2", + }, + }), + ); + }); +}); diff --git a/test/tools/painter.test.ts b/test/tools/painter.test.ts new file mode 100644 index 000000000..1cf405fd7 --- /dev/null +++ b/test/tools/painter.test.ts @@ -0,0 +1,363 @@ +import { existsSync } from "node:fs"; +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + getPainterBudget, + resetPainterBudgetSingleton, +} from "../../src/services/image-providers/cost.js"; +import { + encodeMaskPng, + readPngDimensions, +} from "../../src/services/image-providers/masks.js"; +import { + type PainterToolDetails, + buildMaskPath, + isTransientImageError, + painterOutputDir, + painterTimeoutMs, + painterTool, + persistImage, + resolvePainterModel, +} from "../../src/tools/painter.js"; + +const PAINTER_ENV_KEYS = [ + "OPENAI_API_KEY", + "MAESTRO_PAINTER_MODEL", + "MAESTRO_PAINTER_BASE_URL", + "MAESTRO_PAINTER_OUTPUT_DIR", + "MAESTRO_PAINTER_TIMEOUT_MS", +] as const; + +describe("painter: configuration helpers", () => { + const saved: Record = {}; + + beforeEach(() => { + for (const key of PAINTER_ENV_KEYS) { + saved[key] = process.env[key]; + delete process.env[key]; + } + }); + + afterEach(() => { + for (const key of PAINTER_ENV_KEYS) { + if (saved[key] === undefined) delete process.env[key]; + else process.env[key] = saved[key]; + } + }); + + describe("resolvePainterModel", () => { + it("defaults to gpt-image-2", () => { + expect(resolvePainterModel()).toBe("gpt-image-2"); + }); + + it("honors MAESTRO_PAINTER_MODEL", () => { + process.env.MAESTRO_PAINTER_MODEL = "gpt-image-1"; + expect(resolvePainterModel()).toBe("gpt-image-1"); + }); + + it("ignores whitespace-only overrides", () => { + process.env.MAESTRO_PAINTER_MODEL = " "; + expect(resolvePainterModel()).toBe("gpt-image-2"); + }); + }); + + describe("painterTimeoutMs", () => { + it("defaults to 180s", () => { + expect(painterTimeoutMs()).toBe(180_000); + }); + + it("parses an explicit override", () => { + process.env.MAESTRO_PAINTER_TIMEOUT_MS = "30000"; + expect(painterTimeoutMs()).toBe(30_000); + }); + + it("falls back to default on garbage", () => { + process.env.MAESTRO_PAINTER_TIMEOUT_MS = "not-a-number"; + expect(painterTimeoutMs()).toBe(180_000); + }); + + it("falls back to default on non-positive values", () => { + process.env.MAESTRO_PAINTER_TIMEOUT_MS = "0"; + expect(painterTimeoutMs()).toBe(180_000); + }); + }); + + describe("painterOutputDir", () => { + it("defaults into the maestro home assets dir", () => { + const dir = painterOutputDir(); + expect(dir.endsWith(join(".maestro", "assets", "painter"))).toBe(true); + }); + + it("honors MAESTRO_PAINTER_OUTPUT_DIR", () => { + process.env.MAESTRO_PAINTER_OUTPUT_DIR = "/tmp/painter-override"; + expect(painterOutputDir()).toBe("/tmp/painter-override"); + }); + }); + + describe("isTransientImageError", () => { + it("flags rate-limit and network class failures", () => { + expect(isTransientImageError(new Error("rate limit exceeded"))).toBe( + true, + ); + expect(isTransientImageError(new Error("429 Too Many Requests"))).toBe( + true, + ); + expect( + isTransientImageError(new Error("Service Unavailable (503)")), + ).toBe(true); + expect(isTransientImageError(new Error("ETIMEDOUT"))).toBe(true); + const errnoErr = new Error("connect ECONNREFUSED"); + errnoErr.code = "ECONNREFUSED"; + expect(isTransientImageError(errnoErr)).toBe(true); + }); + + it("does not flag validation or content-policy failures", () => { + expect(isTransientImageError(new Error("Invalid size"))).toBe(false); + expect( + isTransientImageError( + new Error("Painter requires OPENAI_API_KEY to be set"), + ), + ).toBe(false); + expect(isTransientImageError(new Error("content policy"))).toBe(false); + }); + }); +}); + +describe("painter: persistImage", () => { + const tmpRoot = join(tmpdir(), `painter-test-${process.pid}-${Date.now()}`); + + beforeEach(async () => { + process.env.MAESTRO_PAINTER_OUTPUT_DIR = tmpRoot; + await mkdir(tmpRoot, { recursive: true }); + }); + + afterEach(async () => { + delete process.env.MAESTRO_PAINTER_OUTPUT_DIR; + await rm(tmpRoot, { recursive: true, force: true }); + }); + + it("writes the decoded bytes and returns the absolute path", async () => { + const bytes = Buffer.from([1, 2, 3, 4]); + const { path } = await persistImage(bytes, "png"); + expect(existsSync(path)).toBe(true); + expect(path.endsWith(".png")).toBe(true); + }); + + it("creates the output directory if it does not exist", async () => { + const nested = join(tmpRoot, "deeper"); + process.env.MAESTRO_PAINTER_OUTPUT_DIR = nested; + const { path } = await persistImage(Buffer.from([0]), "png"); + expect(existsSync(path)).toBe(true); + }); +}); + +describe("painter: buildMaskPath", () => { + const tmpRoot = join(tmpdir(), `painter-mask-${process.pid}-${Date.now()}`); + + beforeEach(async () => { + process.env.MAESTRO_PAINTER_OUTPUT_DIR = tmpRoot; + await mkdir(tmpRoot, { recursive: true }); + }); + + afterEach(async () => { + delete process.env.MAESTRO_PAINTER_OUTPUT_DIR; + await rm(tmpRoot, { recursive: true, force: true }); + }); + + it("synthesizes a mask sized to the input PNG and reports the png-header source", async () => { + const inputPng = encodeMaskPng( + { width: 20, height: 10 }, + { x: 0, y: 0, width: 1, height: 1 }, + ); + const inputPath = join(tmpRoot, "input.png"); + await writeFile(inputPath, inputPng); + + const { maskPath, dimensions, source } = await buildMaskPath([inputPath], { + x: 5, + y: 2, + width: 4, + height: 3, + }); + + expect(source).toBe("png-header"); + expect(dimensions).toEqual({ width: 20, height: 10 }); + expect(existsSync(maskPath)).toBe(true); + expect(readPngDimensions(await readFile(maskPath))).toEqual({ + width: 20, + height: 10, + }); + }); + + it("writes the mask into the configured output directory", async () => { + const inputPng = encodeMaskPng( + { width: 8, height: 8 }, + { x: 0, y: 0, width: 1, height: 1 }, + ); + const inputPath = join(tmpRoot, "input.png"); + await writeFile(inputPath, inputPng); + + const { maskPath } = await buildMaskPath([inputPath], { + x: 1, + y: 1, + width: 2, + height: 2, + }); + + expect(maskPath.startsWith(tmpRoot)).toBe(true); + expect(maskPath.endsWith(".png")).toBe(true); + }); + + it("rejects when no input images are provided", async () => { + await expect( + buildMaskPath([], { x: 0, y: 0, width: 1, height: 1 }), + ).rejects.toThrow(/at least one input image/); + }); +}); + +describe("painter: provider selection", () => { + const saved: Record = {}; + + beforeEach(() => { + for (const k of [ + "OPENAI_API_KEY", + "FAL_KEY", + "MAESTRO_PAINTER_PROVIDER", + "MAESTRO_PAINTER_MAX_COST_CENTS", + ]) { + saved[k] = process.env[k]; + delete process.env[k]; + } + resetPainterBudgetSingleton(); + }); + + afterEach(() => { + for (const [k, v] of Object.entries(saved)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + resetPainterBudgetSingleton(); + }); + + it("defaults to the OpenAI provider and requires OPENAI_API_KEY", async () => { + await expect( + painterTool.execute("c5", { mode: "generate", prompt: "x" }), + ).rejects.toThrow(/OPENAI_API_KEY/); + }); + + it("routes to FLUX when MAESTRO_PAINTER_PROVIDER=flux and requires FAL_KEY", async () => { + process.env.MAESTRO_PAINTER_PROVIDER = "flux"; + await expect( + painterTool.execute("c6", { mode: "generate", prompt: "x" }), + ).rejects.toThrow(/FAL_KEY/); + }); + + it("rejects edit when the selected provider does not support it", async () => { + process.env.MAESTRO_PAINTER_PROVIDER = "flux"; + process.env.FAL_KEY = "fal-test"; + await expect( + painterTool.execute("c7", { + mode: "edit", + prompt: "x", + images: ["/tmp/a.png"], + }), + ).rejects.toThrow(/does not support editing/); + }); + + it("refunds the reserved budget when the call fails after reservation", async () => { + // flux + edit: budget reserves for the request, then the supports guard + // throws — the refund path must release the reservation so failed calls + // do not permanently consume the ceiling. + process.env.MAESTRO_PAINTER_PROVIDER = "flux"; + process.env.FAL_KEY = "fal-test"; + process.env.MAESTRO_PAINTER_MAX_COST_CENTS = "100"; + process.env.MAESTRO_PAINTER_PRICE_TABLE = JSON.stringify({ + "flux|*|*": 30, + }); + resetPainterBudgetSingleton(); + await expect( + painterTool.execute("c8", { + mode: "edit", + prompt: "x", + images: ["/tmp/a.png"], + }), + ).rejects.toThrow(/does not support editing/); + expect(getPainterBudget().cumulativeCents).toBe(0); + }); +}); + +describe("painter: input guards", () => { + const savedKey = process.env.OPENAI_API_KEY; + + beforeEach(() => { + delete process.env.OPENAI_API_KEY; + }); + + afterEach(() => { + if (savedKey === undefined) delete process.env.OPENAI_API_KEY; + else process.env.OPENAI_API_KEY = savedKey; + }); + + it("rejects when OPENAI_API_KEY is not set", async () => { + await expect( + painterTool.execute("call_1", { mode: "generate", prompt: "an icon" }), + ).rejects.toThrow(/OPENAI_API_KEY/); + }); + + it("rejects edit mode without any input images", async () => { + process.env.OPENAI_API_KEY = "sk-test"; + await expect( + painterTool.execute("call_2", { mode: "edit", prompt: "redact" }), + ).rejects.toThrow(/images/); + }); + + it("rejects an invalid mode at the schema layer", async () => { + process.env.OPENAI_API_KEY = "sk-test"; + await expect( + painterTool.execute("call_3", { + mode: "upscale", + prompt: "x", + } as unknown as { mode: "generate"; prompt: string }), + ).rejects.toThrow(/Validation failed/); + }); + + it("enforces the cost ceiling before any API call", async () => { + process.env.OPENAI_API_KEY = "sk-test"; + process.env.MAESTRO_PAINTER_MAX_COST_CENTS = "1"; + process.env.MAESTRO_PAINTER_PRICE_TABLE = JSON.stringify({ + "gpt-image-2|1024x1024|auto": 8, + }); + // The budget is a process singleton that reads ceiling env at construction; + // reset so this test's env is honored. + resetPainterBudgetSingleton(); + await expect( + painterTool.execute("call_4", { + mode: "generate", + prompt: "icon", + size: "1024x1024", + }), + ).rejects.toThrow(/exceeded/); + resetPainterBudgetSingleton(); + }); + + it("declares a stable name, schema, and summary hooks", () => { + expect(painterTool.name).toBe("painter"); + expect(painterTool.parameters).toBeDefined(); + expect(typeof painterTool.getToolUseSummary).toBe("function"); + expect(painterTool.getToolUseSummary?.({ mode: "generate" })).toBe( + "painter generate", + ); + }); + + // Sentinel to keep the details type honest if the schema grows. + it("uses PainterToolDetails with the persisted paths", () => { + const details: PainterToolDetails = { + provider: "openai", + model: "gpt-image-2", + mode: "generate", + paths: ["/tmp/x.png"], + }; + expect(details.paths).toHaveLength(1); + }); +}); diff --git a/test/tools/tools.test.ts b/test/tools/tools.test.ts index 16547543a..6d2603a9b 100644 --- a/test/tools/tools.test.ts +++ b/test/tools/tools.test.ts @@ -1696,6 +1696,7 @@ describe("codingTools bundle", () => { "read", "list", "oracle", + "painter", "find", "extract_document", "search", diff --git a/test/utils/terminal-image.test.ts b/test/utils/terminal-image.test.ts new file mode 100644 index 000000000..d129d28fa --- /dev/null +++ b/test/utils/terminal-image.test.ts @@ -0,0 +1,134 @@ +import { describe, expect, it } from "vitest"; +import { + type TerminalImageSupport, + detectTerminalImageSupport, + encodeInlineImage, + encodeItermInline, + encodeKittyInline, +} from "../../src/utils/terminal-image.js"; + +const SAMPLE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x00, 0x00]); + +describe("terminal-image: detection", () => { + it("detects iTerm.app", () => { + expect(detectTerminalImageSupport({ TERM_PROGRAM: "iTerm.app" })).toBe( + "iterm", + ); + }); + + it("detects WezTerm via the 1337 protocol", () => { + expect(detectTerminalImageSupport({ TERM_PROGRAM: "WezTerm" })).toBe( + "iterm", + ); + }); + + it("detects kitty by TERM", () => { + expect(detectTerminalImageSupport({ TERM: "xterm-kitty" })).toBe("kitty"); + }); + + it("detects kitty by TERM_PROGRAM", () => { + expect(detectTerminalImageSupport({ TERM_PROGRAM: "kitty" })).toBe("kitty"); + }); + + it("detects sixel only on an explicit TERM signal", () => { + expect(detectTerminalImageSupport({ TERM: "xterm-256color-sixel" })).toBe( + "sixel", + ); + }); + + it("returns none for an unsupported terminal", () => { + expect(detectTerminalImageSupport({ TERM: "xterm-256color" })).toBe("none"); + }); + + it("returns none with no env hints", () => { + expect(detectTerminalImageSupport({})).toBe("none"); + }); +}); + +describe("terminal-image: iTerm2 (OSC 1337)", () => { + it("wraps the base64 payload with the 1337 File sequence and BEL terminator", () => { + const out = encodeItermInline(SAMPLE); + expect(out.startsWith("\x1b]1337;File=")).toBe(true); + expect(out.endsWith("\x07")).toBe(true); + expect(out).toContain("inline=1"); + expect(out).toContain(SAMPLE.toString("base64")); + }); + + it("applies width/height and base64-encodes the name hint", () => { + const out = encodeItermInline(SAMPLE, { + width: 40, + height: "auto", + name: "icon.png", + }); + expect(out).toContain("width=40"); + expect(out).toContain("height=auto"); + expect(out).toContain(`name=${btoa("icon.png")}`); + }); +}); + +describe("terminal-image: kitty graphics", () => { + it("emits a transmit chunk and continuation chunks under the ST terminator", () => { + const out = encodeKittyInline(SAMPLE); + // First chunk carries the action/format headers. + expect(out.startsWith("\x1b_Ga=T,f=100,t=f;")).toBe(true); + // Every chunk is terminated with String Terminator (ESC \). + const parts = out.split("\x1b_G").slice(1); + for (const part of parts) { + expect(part.endsWith("\x1b\\")).toBe(true); + } + }); + + it("splits large payloads into multiple m=1 continuation chunks", () => { + // 10000 bytes -> base64 ~13334 chars -> several 4096-char chunks. + const big = Buffer.alloc(10000, 0x41); + const out = encodeKittyInline(big); + const segments = out.split("\x1b_G").slice(1); + const transmit = segments.filter((s) => + s.startsWith("a=T,f=100,t=f;"), + ).length; + const continuations = segments.filter((s) => s.startsWith("m=1;")).length; + expect(transmit).toBe(1); + expect(continuations).toBeGreaterThanOrEqual(1); + // The concatenated base64 across chunks reconstructs the source payload. + const reassembled = segments + .map((p) => { + const body = p.endsWith("\x1b\\") ? p.slice(0, -2) : p; + return body.split(";").pop() ?? ""; + }) + .join(""); + expect(Buffer.from(reassembled, "base64")).toEqual(big); + }); + + it("emits a single transmit chunk for an empty payload", () => { + const out = encodeKittyInline(Buffer.alloc(0)); + expect(out).toBe("\x1b_Ga=T,f=100,t=f;\x1b\\"); + }); +}); + +describe("terminal-image: encodeInlineImage dispatch", () => { + it("routes to the iTerm2 encoder", () => { + const out = encodeInlineImage(SAMPLE, "iterm"); + expect(out.startsWith("\x1b]1337;File=")).toBe(true); + }); + + it("routes to the kitty encoder", () => { + const out = encodeInlineImage(SAMPLE, "kitty"); + expect(out.startsWith("\x1b_Ga=T,f=100,t=f;")).toBe(true); + }); + + it("returns empty for none", () => { + expect(encodeInlineImage(SAMPLE, "none")).toBe(""); + }); + + it("returns empty for sixel (rasterization not implemented)", () => { + expect(encodeInlineImage(SAMPLE, "sixel")).toBe(""); + }); + + it("defaults to the live environment detection", () => { + const support: TerminalImageSupport = + process.env.TERM_PROGRAM === "iTerm.app" ? "iterm" : "none"; + const out = encodeInlineImage(SAMPLE, support); + if (support === "none") expect(out).toBe(""); + else expect(out.length).toBeGreaterThan(0); + }); +}); diff --git a/test/web/chat-handler-profile.test.ts b/test/web/chat-handler-profile.test.ts index b036a928c..c7a8b07c8 100644 --- a/test/web/chat-handler-profile.test.ts +++ b/test/web/chat-handler-profile.test.ts @@ -122,6 +122,77 @@ describe("chat handler profile threading", () => { vi.doUnmock("../../src/agent/user-prompt-runtime.js"); vi.resetModules(); vi.restoreAllMocks(); + vi.unstubAllEnvs(); + }); + + it("emits identical Oracle experiment policy metadata over SSE and websocket", async () => { + vi.stubEnv("MAESTRO_ORACLE_EXPERIMENT_ID", "oracle-july"); + vi.stubEnv("MAESTRO_ORACLE_EXPERIMENT_ALLOCATION", "1"); + vi.stubEnv("MAESTRO_ORACLE_EXPERIMENT_CONTROL_VERSION", "oracle-v1"); + vi.stubEnv("MAESTRO_ORACLE_EXPERIMENT_TREATMENT_VERSION", "oracle-v2"); + const runUserPromptWithRecovery = vi.fn(async () => {}); + const { handleChat, handleChatWebSocket } = + await importChatHandlersWithMock(runUserPromptWithRecovery); + const context: Partial = { + createAgent: async () => createMockAgent(), + getRegisteredModel: async () => mockModel, + defaultApprovalMode: "prompt", + defaultProvider: "anthropic", + defaultModelId: mockModel.id, + corsHeaders: cors, + }; + + const httpReq = new PassThrough() as MockPassThrough; + httpReq.method = "POST"; + httpReq.url = "/api/chat"; + httpReq.headers = {}; + httpReq.end( + JSON.stringify({ messages: [{ role: "user", content: "hi" }] }), + ); + const res = makeRes(); + await handleChat( + httpReq as unknown as IncomingMessage, + res as unknown as ServerResponse, + context as WebServerContext, + ); + const sseReceipt = res.body + .split("\n") + .find( + (line) => + line.startsWith("data: {") && line.includes("routing_receipt"), + ); + + const wsReq = new PassThrough() as MockPassThrough; + wsReq.method = "GET"; + wsReq.url = "/api/chat/ws"; + wsReq.headers = { host: "localhost" }; + const ws = new MockWebSocket(); + handleChatWebSocket( + ws as unknown as Parameters[0], + wsReq as unknown as IncomingMessage, + context as WebServerContext, + ); + ws.emit( + "message", + JSON.stringify({ messages: [{ role: "user", content: "hi" }] }), + ); + await vi.waitFor(() => + expect( + ws.sent.some((payload) => payload.includes("routing_receipt")), + ).toBe(true), + ); + + const httpExperiment = JSON.parse(sseReceipt!.slice("data: ".length)) + .receipt.experiment; + const wsExperiment = JSON.parse( + ws.sent.find((payload) => payload.includes("routing_receipt"))!, + ).receipt.experiment; + expect(httpExperiment).toEqual({ + experimentId: "oracle-july", + arm: "treatment", + policyVersion: "oracle-v2", + }); + expect(wsExperiment).toEqual(httpExperiment); }); it("passes the server profile into SSE prompt recovery", async () => { diff --git a/test/web/session-serialization.test.ts b/test/web/session-serialization.test.ts index d904e9347..0beafaa55 100644 --- a/test/web/session-serialization.test.ts +++ b/test/web/session-serialization.test.ts @@ -56,7 +56,32 @@ const mockOpenAiModel: RegisteredModel = { isLocal: false, }; +const routingReceipt = { + decisionId: "decision-1", + requestedProfile: "medium", + source: "request" as const, + resolvedProfileId: "medium-v1", + resolvedProfileVersion: 1, + provider: "anthropic", + model: "claude-sonnet-4-5", + reasoningEffort: "medium", + createdAt: "2026-07-14T12:00:00.000Z", +}; + describe("session serialization", () => { + it("round-trips routing receipts on assistant messages", () => { + const composer: ComposerMessage = { + role: "assistant", + content: "Done", + routingReceipt, + }; + const [app] = convertComposerMessageToApp(composer, mockModel); + expect((app as AssistantMessage).routingReceipt).toEqual(routingReceipt); + expect(convertAppMessageToComposer(app!).routingReceipt).toEqual( + routingReceipt, + ); + }); + it("converts app messages with thinking and tools to composer format", () => { const userMessage: AppMessage = { role: "user", diff --git a/test/web/sse-session.test.ts b/test/web/sse-session.test.ts index b067de105..08a490eb0 100644 --- a/test/web/sse-session.test.ts +++ b/test/web/sse-session.test.ts @@ -53,6 +53,27 @@ describe("SseSession", () => { expect(res.chunks.some((c: string) => c.includes("heartbeat"))).toBe(true); }); + it("writes routing receipts as first-class stream events", () => { + const res = createRes(); + const session = new SseSession(res as unknown as ServerResponse); + + session.sendRoutingReceipt({ + decisionId: "decision-1", + requestedProfile: "high", + source: "session", + resolvedProfileId: "high-v1", + resolvedProfileVersion: 1, + provider: "anthropic", + model: "claude-opus-4-6", + reasoningEffort: "xhigh", + createdAt: "2026-07-14T12:00:00.000Z", + }); + + expect(res.chunks).toContainEqual( + expect.stringContaining('"type":"routing_receipt"'), + ); + }); + it("records skipped writes after disconnect", () => { const res = createRes(); res.writable = false;