Merge pull request #2046 from Automattic/feat/2043-external-mysql-provider

feat(runtime-services): support isolated external MySQL providers

Author: chubes4

package.json

@@ -247,7 +247,7 @@
     "test:generic-ability-runtime-run": "tsx tests/generic-ability-runtime-run.test.ts",
     "test:provider-runtime-contracts": "tsx tests/provider-runtime-contracts.test.ts",
     "test:runtime-requirements-readiness": "tsx tests/runtime-requirements-readiness.test.ts",
-    "test:runtime-services": "tsx tests/runtime-services.test.ts",
+    "test:runtime-services": "tsx tests/runtime-services.test.ts && tsx tests/external-mysql-runtime-service.test.ts",
     "test:runtime-services-lifecycle": "tsx tests/runtime-services-lifecycle.test.ts",
     "test:disposable-mysql-mysqli-e2e": "tsx tests/disposable-mysql-mysqli.integration.test.ts",
     "test:runtime-contract-manifest": "tsx tests/runtime-contract-manifest.test.ts",

packages/cli/src/commands/recipe-run-types.ts

@@ -23,6 +23,7 @@ export interface RecipeRunOptions {
   timeoutMs: number
   adversarialReplayPath?: string
   policy?: RuntimePolicy
+  externalServiceWritesApproved: boolean
   json: boolean
   summary: boolean
   dryRun: boolean

packages/cli/src/commands/recipe-run.ts

@@ -11,7 +11,7 @@ import { parsePreviewBind, parsePreviewHoldSeconds, parsePreviewLease, parsePrev
 import { dryRunRecipe, planWorkspaceRecipe, recipeDryRunSiteSeeds } from "../recipe-dry-run.js"
 import { appendRecipeRuntimeEvidence, collectAndFinalizeFailedRecipeArtifacts, collectRecipeRuntimeArtifacts, finalizeAgentSandboxEvidence, finalizeRecipeArtifactEvidence, recipeAgentResultFailure, recipeArtifactEvidenceFailure, recipeReplayStatusOutput, recipeVerifyStepFailure } from "../recipe-evidence.js"
 import { recipeExternalServiceBoundarySummaries } from "../recipe-external-services.js"
-import { resolveRecipeSecretEnv } from "../recipe-secret-env.js"
+import { mergeRecipeSecretEnvSummary, resolveRecipeSecretEnv } from "../recipe-secret-env.js"
 import type { PreparedRuntimeBackendPackage } from "../recipe-backend-package.js"
 import { cleanupRecipePreparedSources, recipeBlueprintWithBootActivePlugins, recipeExtraPlugins, type PreparedDependencyOverlay, type PreparedExtraPlugin, type PreparedRuntimeOverlay, type PreparedStagedFile, type PreparedWorkspaceMount } from "../recipe-sources.js"
 import { loadWorkspaceRecipe, recipePolicy, recipeWorkflowSteps, validateRecipeRuntimePolicy, validateWorkspaceRecipe, type RecipeWorkflowPhase } from "../recipe-validation.js"
@@ -110,7 +110,7 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe
   await artifactPointer.update({ commandStatus: "queued" })
   const issues = [
     ...await validateWorkspaceRecipe(recipe, recipePath),
-    ...validateRecipeRuntimePolicy(recipe, options.policy),
+    ...validateRecipeRuntimePolicy(recipe, options.policy ?? recipePolicy(recipe)),
   ]
   if (issues.length > 0) {
     const failure = {
@@ -140,7 +140,8 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe
   }
   const secretEnvResolution = resolveRecipeSecretEnv(recipe.inputs?.secretEnv ?? [], { field: "--secret-env name" })
   const secretEnv = secretEnvResolution.values
-  const effectivePolicy = Object.keys(secretEnv).length > 0 ? { ...policy, secrets: "connector-scoped" as const } : policy
+  let secretEnvSummary = secretEnvResolution.summary
+  let effectivePolicy = Object.keys(secretEnv).length > 0 ? { ...policy, secrets: "connector-scoped" as const } : policy
   let workspaceMounts: PreparedWorkspaceMount[] = []
   let extraPlugins: PreparedExtraPlugin[] = []
   let dependencyOverlays: PreparedDependencyOverlay[] = []
@@ -194,10 +195,22 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe
     managedServices = await phaseTracker.run("provision_runtime_services", { services: recipe.inputs?.services?.map(({ id, kind }) => ({ id, kind })) ?? [] }, async () => await provisionRuntimeServicesForRecipe(
       recipe.inputs?.services ?? [],
       async (provisioning) => await awaitRecipe("runtime-services.provision", provisioning),
-      { signal: interruption?.signal, onEvidence: (evidence) => { serviceEvidence = evidence } },
+      {
+        signal: interruption?.signal,
+        policy,
+        externalServices: recipe.inputs?.externalServices ?? [],
+        externalServiceWritesApproved: options.externalServiceWritesApproved,
+        onEvidence: (evidence) => { serviceEvidence = evidence },
+      },
     ))
     serviceEvidence = managedServices.evidence
     Object.assign(runtimeEnv, managedServices.env)
+    for (const [name, value] of Object.entries(managedServices.secretEnv)) {
+      delete runtimeEnv[name]
+      secretEnv[name] = value
+    }
+    secretEnvSummary = mergeRecipeSecretEnvSummary(secretEnvSummary, Object.keys(managedServices.secretEnv))
+    if (Object.keys(managedServices.secretEnv).length > 0) effectivePolicy = { ...policy, secrets: "connector-scoped" }
     const runtimeEnvironment = {
       kind: "wordpress" as const,
       name: plan.runtime.name,
@@ -353,7 +366,7 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe
       if (declaredArtifactFailure) {
         throw declaredArtifactFailure
       }
-      const recipeEvidence = await finalizeRecipeArtifactEvidence(artifacts, recipe, workspaceMounts, stagedFiles, effectivePolicy, secretEnvResolution.summary)
+      const recipeEvidence = await finalizeRecipeArtifactEvidence(artifacts, recipe, workspaceMounts, stagedFiles, effectivePolicy, secretEnvSummary)
       const agentEvidence = await finalizeAgentSandboxEvidence(artifacts, executions)
       Object.assign(recipeEvidence, agentEvidence)
       markRecipeArtifactsFinalized(interruption, true)
@@ -374,7 +387,7 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe
       declaredArtifacts = await collectRecipeDeclaredArtifacts(recipe, runtime)
       await materializeTypedRecipeDeclaredArtifacts(artifacts, declaredArtifacts)
       await appendRecipeRuntimeEvidence(artifacts, recipeRuntimeEvidenceFiles(fixtureDatabases, distributionSetupArtifacts, distributionStartupProbes, probes, declaredArtifacts))
-      evidence = await finalizeRecipeArtifactEvidence(artifacts, recipe, workspaceMounts, stagedFiles, effectivePolicy, secretEnvResolution.summary)
+      evidence = await finalizeRecipeArtifactEvidence(artifacts, recipe, workspaceMounts, stagedFiles, effectivePolicy, secretEnvSummary)
       const previewAgentEvidence = await finalizeAgentSandboxEvidence(artifacts, executions)
       Object.assign(evidence, previewAgentEvidence)
     }
@@ -489,7 +502,7 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe
         workspaceMounts,
         stagedFiles,
         policy: effectivePolicy,
-        secretEnv: secretEnvResolution.summary,
+        secretEnv: secretEnvSummary,
         executions,
         interruption,
       }))
@@ -714,7 +727,7 @@ async function validateRecipe(options: RecipeValidateOptions): Promise<RecipeVal
     const recipe = await loadWorkspaceRecipe(recipePath)
     const issues = [
       ...await validateWorkspaceRecipe(recipe, recipePath),
-      ...validateRecipeRuntimePolicy(recipe, options.policy),
+      ...validateRecipeRuntimePolicy(recipe, options.policy ?? recipePolicy(recipe)),
     ]
 
     return {
@@ -765,7 +778,7 @@ function distributionRuntimeEnv(recipe: WorkspaceRecipe): Record<string, string>
 }
 
 function parseRecipeRunOptions(args: string[]): RecipeRunOptions {
-  const parsed = parseCommandOptions(args, new Set(["--json", "--summary", "--summary-only", "--dry-run", "--preview-hold-blocking", "--preview-lease", "--preview-lease-child"]))
+  const parsed = parseCommandOptions(args, new Set(["--json", "--summary", "--summary-only", "--dry-run", "--preview-hold-blocking", "--preview-lease", "--preview-lease-child", "--approve-external-service-writes"]))
   if (parsed.positionals.length > 0) {
     throw new Error(`Invalid argument: ${parsed.positionals[0]}`)
   }
@@ -776,6 +789,7 @@ function parseRecipeRunOptions(args: string[]): RecipeRunOptions {
     previewHoldBlocking: parsed.options.get("--preview-hold-blocking") === true,
     previewLeaseRequested: parsed.options.get("--preview-lease") === true,
     previewLeaseChild: parsed.options.get("--preview-lease-child") === true,
+    externalServiceWritesApproved: parsed.options.get("--approve-external-service-writes") === true,
     timeoutMs: DEFAULT_RECIPE_RUN_TIMEOUT_MS,
   }
   for (const [name, value] of parsed.options) {

packages/cli/src/output.ts

@@ -431,6 +431,8 @@ Options:
                          wp-codebox/preview-lease/v1 envelope for public/local URL, expiry, alignment, and handoff metadata.
   --timeout <duration>  Maximum live recipe-run duration before emitting a structured timeout failure. Defaults to 25m.
   --policy <json|file> Runtime policy JSON or path to a JSON file. For recipe-run and recipe validate, this overrides the recipe-derived runtime policy and must include every command required by the recipe setup, probes, and workflow.
+  --approve-external-service-writes
+                       Explicitly approve short-lived managed writes to declared external-service boundaries when policy.approvals is on-write.
   --dry-run            Validate recipe-run and emit a resolved JSON plan without booting Playground or writing temp workspaces.
   --json               Emit machine-readable JSON.
 

packages/cli/src/recipe-evidence.ts

@@ -1344,13 +1344,7 @@ async function buildRecipeRunAttestation(args: {
         resultRef: artifactVerificationRef,
       }),
     },
-    secretEnvelope: {
-      schema: "wp-codebox/redacted-secret-envelope/v1",
-      provided: args.secretEnv.some((entry) => entry.status === "available"),
-      count: args.secretEnv.filter((entry) => entry.status === "available").length,
-      secrets: [...args.secretEnv].sort((a, b) => a.name.localeCompare(b.name)),
-      redaction: "names-only",
-    },
+    secretEnvelope: recipeSecretEnvelope(args.secretEnv),
     externalServices: {
       schema: "wp-codebox/external-service-boundaries-attestation/v1",
       boundaries: recipeExternalServiceBoundarySummaries(args.recipe),
@@ -1378,6 +1372,22 @@ async function buildRecipeRunAttestation(args: {
   }
 }
 
+export function recipeSecretEnvelope(secretEnv: readonly RecipeSecretEnvSummaryEntry[]): {
+  schema: "wp-codebox/redacted-secret-envelope/v1"
+  provided: boolean
+  count: number
+  secrets: Array<{ name: string; status: RecipeSecretEnvSummaryEntry["status"]; source?: string }>
+  redaction: "names-only"
+} {
+  return {
+    schema: "wp-codebox/redacted-secret-envelope/v1",
+    provided: secretEnv.some((entry) => entry.status === "available"),
+    count: secretEnv.filter((entry) => entry.status === "available").length,
+    secrets: [...secretEnv].sort((a, b) => a.name.localeCompare(b.name)),
+    redaction: "names-only",
+  }
+}
+
 function enforcedPolicyField(value: RuntimePolicy[keyof RuntimePolicy]): RunAttestationPolicyField {
   return { value, enforcement: "enforced" }
 }

packages/cli/src/recipe-secret-env.ts

@@ -54,6 +54,15 @@ export function resolveRecipeSecretEnv(names: readonly string[], options: Resolv
   return { values, summary }
 }
 
+export function mergeRecipeSecretEnvSummary(summary: readonly RecipeSecretEnvSummaryEntry[], generatedNames: readonly string[]): RecipeSecretEnvSummaryEntry[] {
+  const entries = new Map(summary.map((entry) => [entry.name, { ...entry }]))
+  for (const name of generatedNames) {
+    assertRuntimeEnvName(name, "managed runtime service secret env")
+    entries.set(name, { name, status: "available", source: "managed-runtime-service" })
+  }
+  return [...entries.values()].sort((left, right) => left.name.localeCompare(right.name))
+}
+
 export function defaultRecipeSecretEnvProviders(source: Record<string, string | undefined> = process.env): RecipeSecretEnvProvider[] {
   return [
     directProcessEnvSecretProvider,

packages/cli/src/recipe-validation.ts

@@ -536,6 +536,26 @@ export function validateRecipeRuntimePolicy(recipe: WorkspaceRecipe, policy: Run
     }
   }
 
+  const externalMysqlServices = (recipe.inputs?.services ?? []).filter((service) => service.kind === "mysql" && service.configuration?.provider === "external")
+  if (externalMysqlServices.length > 0) {
+    if (policy.network === "deny") {
+      issues.push({ code: "runtime-policy-external-service-network-denied", path: "$.policy.network", message: "External MySQL runtime services require network access to their declared external-service hosts." })
+    }
+    if (policy.approvals !== "on-write") {
+      issues.push({ code: "runtime-policy-external-service-approval-required", path: "$.policy.approvals", message: "External MySQL runtime services require approvals=on-write and explicit write approval at execution time." })
+    }
+    if (typeof policy.network === "object") {
+      for (const service of externalMysqlServices) {
+        const boundary = recipe.inputs?.externalServices?.find((candidate) => candidate.id === service.configuration?.externalService)
+        for (const host of boundary?.allowedHosts ?? []) {
+          if (!policyHostListIncludes(policy.network.allowHosts, host)) {
+            issues.push({ code: "runtime-policy-external-service-host-denied", path: "$.policy.network.allowHosts", message: `Runtime network policy must allow external-service host: ${host}` })
+          }
+        }
+      }
+    }
+  }
+
   return issues
 }
 
@@ -733,17 +753,34 @@ export async function validateWorkspaceRecipeSemantics(recipe: WorkspaceRecipe,
 
 function validateRecipeRuntimeServices(recipe: WorkspaceRecipe, addIssue: (code: string, path: string, message: string) => void): void {
   const ids = new Set<string>()
-  const environment = new Set<string>([
+  const exposedEnvironment = new Set<string>([
     ...Object.keys(recipe.distribution?.env ?? {}),
     ...Object.keys(recipe.inputs?.runtimeEnv ?? {}),
     ...(recipe.inputs?.secretEnv ?? []),
   ])
+  const environment = new Set(exposedEnvironment)
   for (const [index, service] of (recipe.inputs?.services ?? []).entries()) {
     const path = `$.inputs.services[${index}]`
     if (!/^[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(service.id)) addIssue("invalid-runtime-service-id", `${path}.id`, "Runtime service ids must be stable identifiers.")
     if (ids.has(service.id)) addIssue("duplicate-runtime-service-id", `${path}.id`, `Runtime service ids must be unique: ${service.id}`)
     ids.add(service.id)
     if (!["mysql", "redis", "smtp", "http"].includes(service.kind)) addIssue("unsupported-runtime-service-kind", `${path}.kind`, `Unsupported managed runtime service kind: ${service.kind}`)
+    if (service.configuration?.provider === "external") {
+      if (service.kind !== "mysql") addIssue("unsupported-runtime-service-provider", `${path}.configuration.provider`, "The external provider supports only MySQL-compatible services.")
+      const boundary = recipe.inputs?.externalServices?.find((candidate) => candidate.id === service.configuration?.externalService)
+      if (!boundary) addIssue("missing-runtime-service-external-boundary", `${path}.configuration.externalService`, "External MySQL services must reference an inputs.externalServices boundary.")
+      else {
+        if ((boundary.allowedHosts ?? []).length === 0) addIssue("missing-runtime-service-host-allowlist", `${path}.configuration.externalService`, "External MySQL service boundaries must explicitly allow at least one host.")
+        if (boundary.writes !== "allowed-with-approval") addIssue("runtime-service-writes-not-approved", `${path}.configuration.externalService`, "External MySQL service boundaries must declare writes=allowed-with-approval.")
+      }
+      for (const field of ["hostEnv", "portEnv", "usernameEnv", "passwordEnv"] as const) {
+        const name = service.configuration[field]
+        if (name && exposedEnvironment.has(name)) addIssue("runtime-service-admin-env-exposed", `${path}.configuration.${field}`, `External service administration environment must remain host-only: ${name}`)
+      }
+      for (const field of ["image", "rootAuthentication", "foreignKeyTargetPolicy"] as const) {
+        if (service.configuration[field] !== undefined) addIssue("unsupported-external-runtime-service-option", `${path}.configuration.${field}`, `External MySQL services do not support ${field}.`)
+      }
+    }
     const supportedOutputs: Record<string, RegExp> = {
       mysql: /^(host|port|username|password|database)$/,
       redis: /^(host|port|url)$/,
@@ -759,6 +796,15 @@ function validateRecipeRuntimeServices(recipe: WorkspaceRecipe, addIssue: (code:
   }
 }
 
+function policyHostListIncludes(policyHosts: readonly string[], boundaryHost: string): boolean {
+  const normalizedBoundary = boundaryHost.trim().toLowerCase().replace(/^https?:\/\//, "").replace(/\/.*$/, "")
+  const boundaryHasPort = /:\d+$/.test(normalizedBoundary)
+  return policyHosts.some((host) => {
+    const normalized = host.trim().toLowerCase().replace(/^https?:\/\//, "").replace(/\/.*$/, "")
+    return normalized === normalizedBoundary || (!boundaryHasPort && normalized.replace(/:\d+$/, "") === normalizedBoundary)
+  })
+}
+
 function validateRecipeExternalServiceBoundaries(recipe: WorkspaceRecipe, addIssue: (code: string, path: string, message: string) => void): void {
   const seenIds = new Set<string>()
   for (const [index, boundary] of (recipe.inputs?.externalServices ?? []).entries()) {