Add code runtime output helpers

Author: RhysSullivan

e2e/scenarios/codemode-output-helpers.test.ts

@@ -0,0 +1,80 @@
+import { expect } from "@effect/vitest";
+import { Effect } from "effect";
+
+import { scenario } from "../src/scenario";
+import { Mcp, Target } from "../src/services";
+
+const PNG_DATA =
+  "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg==";
+const PNG_DATA_URL = `data:image/png;base64,${PNG_DATA}`;
+
+scenario(
+  "Codemode · output helpers emit images, detail metadata, and notifications",
+  {},
+  Effect.gen(function* () {
+    const target = yield* Target;
+    const mcp = yield* Mcp;
+    const identity = yield* target.newIdentity();
+    const session = mcp.session(identity);
+
+    const result = yield* session.call("execute", {
+      code: [
+        'text("helper caption");',
+        `image(${JSON.stringify(PNG_DATA_URL)}, "original");`,
+        `image({ image_url: ${JSON.stringify(PNG_DATA_URL)}, detail: "low" });`,
+        'notify({ message: "rendered previews", data: { count: 2 } });',
+        "return {",
+        '  value: "structured return",',
+        "  toolKeys: {",
+        "    root: Object.keys(tools),",
+        "    describe: Object.keys(tools.describe),",
+        "    executor: Object.keys(tools.executor),",
+        "    sources: Object.keys(tools.executor.sources),",
+        "  },",
+        "};",
+      ].join("\n"),
+    });
+
+    expect(result.ok, `execute completed (got: ${result.text.slice(0, 300)})`).toBe(true);
+
+    const raw = result.raw as {
+      content?: ReadonlyArray<Record<string, unknown>>;
+      structuredContent?: Record<string, unknown>;
+    };
+    const content = raw.content ?? [];
+    expect(result.text, "text() reaches the user-visible MCP text stream").toContain(
+      "helper caption",
+    );
+    expect(result.text, "notify() renders as a distinct visible notification").toContain(
+      "Notification: rendered previews",
+    );
+
+    const images = content.filter((block) => block.type === "image");
+    expect(images, "image() emits two MCP image blocks").toHaveLength(2);
+    expect(images[0], "data URI image is converted to MCP image content").toMatchObject({
+      type: "image",
+      data: PNG_DATA,
+      mimeType: "image/png",
+      _meta: { "codex/imageDetail": "original" },
+    });
+    expect(images[1], "image_url object detail is preserved").toMatchObject({
+      type: "image",
+      data: PNG_DATA,
+      mimeType: "image/png",
+      _meta: { "codex/imageDetail": "low" },
+    });
+
+    expect(raw.structuredContent?.result, "return value stays model-visible").toEqual({
+      value: "structured return",
+      toolKeys: {
+        root: ["search", "describe", "executor"],
+        describe: ["tool"],
+        executor: ["sources"],
+        sources: ["list"],
+      },
+    });
+    expect(raw.structuredContent?.notifications, "notifications are structured too").toEqual([
+      { message: "rendered previews", data: { count: 2 } },
+    ]);
+  }),
+);

packages/core/execution/src/description.ts

@@ -116,14 +116,15 @@ const formatDescription = (connectionEntries: readonly ConnectionInventoryEntry[
     "- Tool calls return a value union: `{ ok: true, data }` for success or `{ ok: false, error: { code, message, status?, details?, retryable? } }` for expected tool/domain failures. Branch on `result.ok`.",
     "- `data` is the upstream payload itself. HTTP-backed tools (OpenAPI) also set `http: { status, headers }` beside `data` — read `result.http?.headers` for pagination (Link) or rate-limit headers.",
     "- Use `emit(value)` to append user-visible output and return `undefined`. Plain values become MCP text content. MCP content blocks are forwarded as-is. `ToolFile` values are rendered by MIME. Emitted output goes to the user, not back to you; the result envelope reports an `emitted` count so you can confirm it landed, but to read a value yourself, `return` it.",
+    "- Prefer explicit output helpers when the content kind is known: `text(value)`, `image(value, detail?)`, `audio(block)`, `file(toolFile)`, `resource(block)`, and `notify(value)`. `image` accepts a base64 data URI string, `{ image_url, detail }`, or a raw MCP image block; `detail` may be `auto`, `low`, `high`, or `original`.",
     '- File-returning tools may return `ToolFile` values: `{ _tag: "ToolFile", name?, mimeType, encoding: "base64", data, byteLength }`. Emit any attachment with `emit(result.data)`.',
     '- To emit MCP-native content directly, pass an MCP content block to `emit(...)`, such as `{ type: "image", data, mimeType }`, `{ type: "audio", data, mimeType }`, `{ type: "text", text }`, `{ type: "resource", resource }`, or `{ type: "resource_link", uri, name, ... }`.',
     "- `emit(ToolFile)` is MIME-based: `image/*` becomes MCP image content, `audio/*` becomes MCP audio content, text-like files become decoded text, and other binary files become embedded MCP resources.",
     "- `return` is only for ordinary structured data. Returning a `ToolFile`, a `ToolResult`, an MCP content block, or a bare base64 string does not emit content to the MCP client.",
     "- Some providers, including Gmail, return attachment bytes without a public URL. To send that attachment to another API from code, decode `ToolFile.data` from base64 and pass the bytes to that API's upload/file input.",
     "- If `tools.search()` returns `hasMore: true` and you didn't find what you need, fetch the next page: `tools.search({ query, offset: nextOffset, limit })`.",
     "- Always use the full address when calling tools: `tools.<integration>.<owner>.<connection>.<tool>(args)`. The `path` returned by `tools.search()` / `tools.describe.tool()` is already the exact path under `tools` — call `tools[path]` rather than guessing segments.",
-    "- The `tools` object is a lazy proxy — `Object.keys(tools)` won't work. Use `tools.search()` or `tools.executor.coreTools.connections.list({})` instead.",
+    "- The `tools` object is a lazy proxy. `Object.keys(tools)` returns only bounded built-in discovery keys (`search`, `describe`, `executor`), not the full API catalog. Use `tools.search()` or `tools.executor.coreTools.connections.list({})` for configured tools.",
     '- Pass an object to system tools, e.g. `tools.search({ query: "..." })`, `tools.executor.coreTools.connections.list({})`, and `tools.describe.tool({ path })`.',
     '- `tools.describe.tool()` returns compact TypeScript shapes. Use `inputTypeScript`, `outputTypeScript`, and `typeScriptDefinitions`. If the path doesn\'t resolve, the result carries `error: { code: "tool_not_found", suggestions }` — use a suggestion instead of retrying the same path.',
     "- For tools that return large collections (e.g. `getStates`, `getAll`), filter results in code rather than calling per-item tools.",

packages/core/execution/src/engine.ts

@@ -10,7 +10,12 @@ import type {
   ElicitationContext,
 } from "@executor-js/sdk/core";
 import { CodeExecutionError } from "@executor-js/codemode-core";
-import type { CodeExecutor, ExecuteResult, SandboxToolInvoker } from "@executor-js/codemode-core";
+import type {
+  CodeExecutor,
+  ExecuteNotification,
+  ExecuteResult,
+  SandboxToolInvoker,
+} from "@executor-js/codemode-core";
 
 import {
   defaultToolDiscoveryProvider,
@@ -79,6 +84,16 @@ export const formatExecuteResult = (
       : null;
 
   const logText = result.logs && result.logs.length > 0 ? result.logs.join("\n") : null;
+  const notifications =
+    result.output
+      ?.filter(
+        (
+          item,
+        ): item is { readonly type: "notification"; readonly notification: ExecuteNotification } =>
+          item.type === "notification",
+      )
+      .map((item) => item.notification) ?? [];
+  const notificationField = notifications.length > 0 ? { notifications } : {};
 
   // `emit()` output is shown to the user, not returned to the model, so a
   // script that only emits comes back with a null result. Acknowledge the
@@ -97,6 +112,7 @@ export const formatExecuteResult = (
         status: "error",
         error: result.error,
         ...emittedField,
+        ...notificationField,
         logs: result.logs ?? [],
       },
       isError: true,
@@ -115,6 +131,7 @@ export const formatExecuteResult = (
       status: "completed",
       result: result.result ?? null,
       ...emittedField,
+      ...notificationField,
       logs: result.logs ?? [],
     },
     isError: false,

packages/hosts/mcp/src/tool-server.ts

@@ -373,13 +373,36 @@ const isContentOutputItem = (
 ): item is { readonly type: "content"; readonly content: ContentBlock } =>
   isRecord(item) && item.type === "content" && isMcpContentBlock(item.content);
 
+const isNotificationOutputItem = (
+  item: ExecuteOutputItem,
+): item is {
+  readonly type: "notification";
+  readonly notification: { readonly message: string; readonly data?: unknown };
+} =>
+  isRecord(item) &&
+  item.type === "notification" &&
+  isRecord(item.notification) &&
+  typeof item.notification.message === "string";
+
+const notificationContent = (notification: {
+  readonly message: string;
+  readonly data?: unknown;
+}): ContentBlock[] => {
+  const dataText =
+    notification.data === undefined ? "" : `\n${JSON.stringify(notification.data, null, 2)}`;
+  return [{ type: "text", text: `Notification: ${notification.message}${dataText}` }];
+};
+
 const outputItemContent = (item: ExecuteOutputItem): ContentBlock[] => {
   if (isFileOutputItem(item)) {
     return outputFileContent(item.file);
   }
   if (isContentOutputItem(item)) {
     return [item.content];
   }
+  if (isNotificationOutputItem(item)) {
+    return notificationContent(item.notification);
+  }
   return [{ type: "text", text: "Invalid execution output item omitted." }];
 };
 

packages/kernel/core/src/types.ts

@@ -27,6 +27,16 @@ export interface SandboxToolInvoker {
 }
 
 /** User-visible output accumulated by sandbox helpers. */
+export type ImageDetail = "auto" | "low" | "high" | "original";
+
+export const IMAGE_DETAIL_META_KEY = "codex/imageDetail";
+export const DEFAULT_IMAGE_DETAIL: ImageDetail = "high";
+
+export type ExecuteNotification = {
+  readonly message: string;
+  readonly data?: unknown;
+};
+
 export type ExecuteOutputItem =
   | {
       readonly type: "file";
@@ -35,8 +45,17 @@ export type ExecuteOutputItem =
   | {
       readonly type: "content";
       readonly content: unknown;
+    }
+  | {
+      readonly type: "notification";
+      readonly notification: ExecuteNotification;
     };
 
+export type CodeExecutionOptions = {
+  readonly onOutput?: (item: ExecuteOutputItem) => void | Promise<void>;
+  readonly onYield?: () => void | Promise<void>;
+};
+
 /** Result of executing code in a sandbox */
 export type ExecuteResult = {
   result: unknown;
@@ -55,7 +74,11 @@ export type ExecuteResult = {
  * `Data.TaggedError` subclass — e.g. `CodeExecutor<WorkerLoaderError>`.
  */
 export interface CodeExecutor<E extends Cause.YieldableError = CodeExecutionError> {
-  execute(code: string, toolInvoker: SandboxToolInvoker): Effect.Effect<ExecuteResult, E>;
+  execute(
+    code: string,
+    toolInvoker: SandboxToolInvoker,
+    options?: CodeExecutionOptions,
+  ): Effect.Effect<ExecuteResult, E>;
 }
 
 /** Accept-anything schema for tools with no input validation */