Emit runtime metadata for parent agent tools (#1766)

Author: chubes4

.github/scripts/run-agent-task/execute-native-agent-task.mjs

@@ -200,6 +200,12 @@ const runnerWorkspaceTools = [
   "create-github-pull-request", "create-github-issue", "comment-github-pull-request",
 ]
 
+function runtimeMetadataForExecutionLocation(executionLocation) {
+  if (executionLocation === "sandbox") return { environment: "runtime_local", capability_scope: "runtime_local" }
+  if (executionLocation === "parent") return { environment: "control_plane", capability_scope: "control_plane" }
+  throw new Error(`Unsupported tool execution location: ${executionLocation}`)
+}
+
 await mkdir(artifactsPath, { recursive: true })
 
 const accessError = accessFailure(request)
@@ -233,7 +239,7 @@ const taskInput = {
     sandbox_tool_policy: {
       schema: "wp-codebox/sandbox-tool-policy/v1",
       version: 1,
-      tools: runnerWorkspaceTools.map((id) => ({ id, runtime_tool_id: id, execution_location: "parent", transport_visibility: "visible", allowed: true })),
+      tools: runnerWorkspaceTools.map((id) => ({ id, runtime_tool_id: id, execution_location: "parent", transport_visibility: "visible", allowed: true, runtime: runtimeMetadataForExecutionLocation("parent") })),
     },
     max_turns: request.limits?.max_turns,
     task_timeout_seconds: Math.ceil(Number(request.limits?.time_budget_ms || 0) / 1000),

packages/runtime-core/src/agent-task-recipe.ts

@@ -1,7 +1,7 @@
 import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"
 import { dirname, join, relative, resolve } from "node:path"
 import { fileURLToPath } from "node:url"
-import type { SandboxToolPolicySnapshot } from "./sandbox-tool-policy.js"
+import { normalizeSandboxToolPolicySnapshot, type SandboxToolPolicySnapshot } from "./sandbox-tool-policy.js"
 import type { StructuredArtifactPayload } from "./structured-artifacts.js"
 import type { TaskInput } from "./task-input.js"
 import { isPlainObject, stringList, stripUndefined } from "./object-utils.js"
@@ -811,14 +811,14 @@ function sandboxToolPolicy(input: AgentTaskRunInput, taskInput: TaskInput): Sand
   const taskPolicy = objectValue(taskInput.sandbox_tool_policy) ?? {}
   const policy = Object.keys(inputPolicy).length > 0 ? inputPolicy : taskPolicy
   if (Object.keys(policy).length > 0) {
-    return policy as unknown as SandboxToolPolicySnapshot
+    return normalizeSandboxToolPolicySnapshot(policy)
   }
-  return {
+  return normalizeSandboxToolPolicySnapshot({
     schema: "wp-codebox/sandbox-tool-policy/v1",
     version: 1,
     tools: [{ id: "deny-all", runtime_tool_id: "deny-all", execution_location: "parent", transport_visibility: "hidden", allowed: false, runtime: { environment: "control_plane", capability_scope: "control_plane" } }],
     metadata: { source: "contained-runtime.agent-task-run.default-deny" },
-  }
+  })
 }
 
 function slugFromPath(source: string): string {

packages/runtime-core/src/sandbox-tool-policy.ts

@@ -153,6 +153,7 @@ export function validateSandboxToolPolicySnapshot(input: unknown): SandboxToolPo
     }
     const id = typeof tool.id === "string" ? tool.id.trim() : ""
     const runtimeToolId = typeof tool.runtime_tool_id === "string" ? tool.runtime_tool_id.trim() : ""
+    const executionLocation = typeof tool.execution_location === "string" ? tool.execution_location.trim() : ""
     const runtime = isPlainObject(tool.runtime) ? tool.runtime : {}
     const runtimeEnvironment = typeof runtime[AGENTS_API_RUNTIME_ENVIRONMENT] === "string" ? runtime[AGENTS_API_RUNTIME_ENVIRONMENT].trim() : ""
     const runtimeCapabilityScope = typeof runtime[AGENTS_API_RUNTIME_CAPABILITY_SCOPE] === "string" ? runtime[AGENTS_API_RUNTIME_CAPABILITY_SCOPE].trim() : ""
@@ -165,11 +166,19 @@ export function validateSandboxToolPolicySnapshot(input: unknown): SandboxToolPo
     if (!runtimeToolId) {
       issues.push({ code: "invalid-tool", field: `${field}.runtime_tool_id`, message: `${field}.runtime_tool_id must be a non-empty string.` })
     }
+    if (executionLocation !== "sandbox" && executionLocation !== "parent") {
+      issues.push({ code: "invalid-tool", field: `${field}.execution_location`, message: `${field}.execution_location must be sandbox or parent.` })
+    }
+    const canonicalRuntime = runtimeMetadataForExecutionLocation(executionLocation)
     if (!runtimeEnvironment) {
       issues.push({ code: "invalid-tool", field: `${field}.runtime.environment`, message: `${field}.runtime.environment must be a non-empty string.` })
+    } else if (canonicalRuntime && runtimeEnvironment !== canonicalRuntime.environment) {
+      issues.push({ code: "invalid-tool", field: `${field}.runtime.environment`, message: `${field}.runtime.environment must be ${canonicalRuntime.environment} for ${executionLocation} tools.` })
     }
     if (!runtimeCapabilityScope) {
       issues.push({ code: "invalid-tool", field: `${field}.runtime.capability_scope`, message: `${field}.runtime.capability_scope must be a non-empty string.` })
+    } else if (canonicalRuntime && runtimeCapabilityScope !== canonicalRuntime.capability_scope) {
+      issues.push({ code: "invalid-tool", field: `${field}.runtime.capability_scope`, message: `${field}.runtime.capability_scope must be ${canonicalRuntime.capability_scope} for ${executionLocation} tools.` })
     }
     if (typeof tool.allowed !== "boolean") {
       issues.push({ code: "invalid-tool", field: `${field}.allowed`, message: `${field}.allowed must be boolean.` })
@@ -260,18 +269,26 @@ export function resolveRuntimeToolAlias(policy: EffectiveRuntimeToolPolicy | San
 }
 
 export function sandboxToolRuntimeMetadata(tool: SandboxToolPolicyTool): Required<Pick<AgentsApiRuntimeToolMetadata, typeof AGENTS_API_RUNTIME_ENVIRONMENT | typeof AGENTS_API_RUNTIME_CAPABILITY_SCOPE>> {
-  const runtime = isPlainObject(tool.runtime) ? tool.runtime : {}
-  const environment = typeof runtime[AGENTS_API_RUNTIME_ENVIRONMENT] === "string"
-    ? runtime[AGENTS_API_RUNTIME_ENVIRONMENT]
-    : ""
-  const capabilityScope = typeof runtime[AGENTS_API_RUNTIME_CAPABILITY_SCOPE] === "string"
-    ? runtime[AGENTS_API_RUNTIME_CAPABILITY_SCOPE]
-    : ""
+  return runtimeMetadataForExecutionLocation(tool.execution_location) ?? {
+    environment: "",
+    capability_scope: "",
+  }
+}
 
-  return {
-    environment,
-    capability_scope: capabilityScope,
+export function runtimeMetadataForExecutionLocation(executionLocation: SandboxToolExecutionLocation): Required<Pick<AgentsApiRuntimeToolMetadata, typeof AGENTS_API_RUNTIME_ENVIRONMENT | typeof AGENTS_API_RUNTIME_CAPABILITY_SCOPE>> | undefined {
+  if (executionLocation === "sandbox") {
+    return {
+      environment: AGENTS_API_RUNTIME_LOCAL,
+      capability_scope: AGENTS_API_RUNTIME_LOCAL,
+    }
+  }
+  if (executionLocation === "parent") {
+    return {
+      environment: AGENTS_API_CONTROL_PLANE,
+      capability_scope: AGENTS_API_CONTROL_PLANE,
+    }
   }
+  return undefined
 }
 
 function runtimeToolDescriptor(tool: SandboxToolPolicyTool): RuntimeToolDescriptor {

tests/agent-task-contracts.test.ts

@@ -122,6 +122,22 @@ assert.equal(stableRunRequestInput.goal, "Run the delegated task.")
 assert.equal(stableRunRequestInput.artifacts_path, "/tmp/stable-run-artifacts")
 assert.deepEqual(stableRunRequestInput.callback_data, { source: "external-orchestrator" })
 
+assert.throws(() => buildAgentTaskRecipe({
+  sandbox_tool_policy: {
+    schema: "wp-codebox/sandbox-tool-policy/v1",
+    version: 1,
+    tools: [{
+      id: "contradictory-parent-tool",
+      runtime_tool_id: "contradictory-parent-tool",
+      execution_location: "parent",
+      transport_visibility: "parent",
+      allowed: true,
+      runtime: { environment: "runtime_local", capability_scope: "runtime_local" },
+    }],
+    metadata: {},
+  },
+}, normalizeTaskInput({ goal: "Reject contradictory tool runtime metadata." }), "latest"), /must be control_plane for parent tools/)
+
 const headlessRequest = normalizeHeadlessAgentTaskRequest({
   schema: HEADLESS_AGENT_TASK_REQUEST_SCHEMA,
   task_input: {

tests/agent-task-reusable-workflow.test.ts

@@ -210,6 +210,20 @@ assert.equal(result.success, true)
 assert.equal(result.runtime_input_path, ".codebox/native-agent-task-input.json")
 assert.deepEqual(result.verification, [])
 assert.doesNotMatch(JSON.stringify(result), /homeboy|agent-task-plan|run-plan/i)
+const nativeTaskInput = JSON.parse(await readFile(join(tmp, ".codebox", "native-agent-task-input.json"), "utf8"))
+const hostedDocsAgentToolSnapshot = [
+  "workspace-read", "workspace-ls", "workspace-grep", "workspace-write", "workspace-edit", "workspace-apply-patch",
+  "workspace-git-status", "workspace-git-diff", "workspace-git-add", "workspace-git-commit", "workspace-git-push",
+  "create-github-pull-request", "create-github-issue", "comment-github-pull-request",
+]
+assert.deepEqual(nativeTaskInput.task_input.sandbox_tool_policy.tools, hostedDocsAgentToolSnapshot.map((id) => ({
+  id,
+  runtime_tool_id: id,
+  execution_location: "parent",
+  transport_visibility: "visible",
+  allowed: true,
+  runtime: { environment: "control_plane", capability_scope: "control_plane" },
+})), "Hosted Docs Agent run 29298164272 must emit canonical control-plane metadata for every parent tool")
 
 const executeNativeAgentTask = new URL("../.github/scripts/run-agent-task/execute-native-agent-task.mjs", import.meta.url).pathname
 const executeAccessCase = async (candidate: Record<string, unknown>, environment: Record<string, string>) => {

tests/runtime-tool-policy.test.ts

@@ -1,5 +1,5 @@
 import assert from "node:assert/strict"
-import { resolveEffectiveRuntimeToolPolicy, resolveRuntimeToolAlias, runtimeToolInputFromSandboxToolPolicy, sandboxAllowedRuntimeToolIds, sandboxToolPolicyFromAllowedTools, toolBridgeFromSandboxToolPolicy, type SandboxToolPolicySnapshot } from "../packages/runtime-core/src/index.js"
+import { assertSandboxToolPolicySnapshot, resolveEffectiveRuntimeToolPolicy, resolveRuntimeToolAlias, runtimeMetadataForExecutionLocation, runtimeToolInputFromSandboxToolPolicy, sandboxAllowedRuntimeToolIds, sandboxToolPolicyFromAllowedTools, toolBridgeFromSandboxToolPolicy, type SandboxToolPolicySnapshot } from "../packages/runtime-core/src/index.js"
 
 const policy: SandboxToolPolicySnapshot = {
   schema: "wp-codebox/sandbox-tool-policy/v1",
@@ -56,6 +56,16 @@ assert.deepEqual(sandboxAllowedRuntimeToolIds(policy), ["client/filesystem-write
 
 assert.deepEqual(runtimeToolInputFromSandboxToolPolicy(policy).allowed_tools, ["client/filesystem-write"])
 assert.deepEqual(runtimeToolInputFromSandboxToolPolicy(policy).runtime_tools.map((tool) => tool.id), ["filesystem-write"])
+assert.deepEqual(runtimeMetadataForExecutionLocation("sandbox"), { environment: "runtime_local", capability_scope: "runtime_local" })
+assert.deepEqual(runtimeMetadataForExecutionLocation("parent"), { environment: "control_plane", capability_scope: "control_plane" })
+assert.equal(runtimeMetadataForExecutionLocation("external"), undefined)
+
+const contradictoryPolicy = structuredClone(policy)
+contradictoryPolicy.tools[1].runtime = { environment: "runtime_local", capability_scope: "runtime_local" }
+assert.throws(() => assertSandboxToolPolicySnapshot(contradictoryPolicy), /must be control_plane for parent tools/)
+const invalidLocationPolicy = structuredClone(policy)
+invalidLocationPolicy.tools[0].execution_location = "external"
+assert.throws(() => assertSandboxToolPolicySnapshot(invalidLocationPolicy), /must be sandbox or parent/)
 
 const canonicalPolicy = sandboxToolPolicyFromAllowedTools(["workspace.read", "workspace.search", "workspace.write", "workspace.edit"], { source: "test" })
 assert.equal(canonicalPolicy.schema, "wp-codebox/sandbox-tool-policy/v1")