fix: close browser environment validation gaps

chubes4 · chubes4 · commit f22389e37804 · 2026-07-26T17:49:18.000Z
diff --git a/packages/cli/src/commands/recipe-run.ts b/packages/cli/src/commands/recipe-run.ts
@@ -110,7 +110,7 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe
   await artifactPointer.update({ commandStatus: "queued" })
   const issues = [
     ...await validateWorkspaceRecipe(recipe, recipePath),
-    ...validateRecipeRuntimePolicy(recipe, options.policy ?? recipePolicy(recipe)),
+    ...validateRecipeRuntimePolicy(recipe, options.policy ?? recipePolicy(recipe, recipeDirectory), recipeDirectory),
   ]
   if (issues.length > 0) {
     const failure = {
@@ -727,7 +727,7 @@ async function validateRecipe(options: RecipeValidateOptions): Promise<RecipeVal
     const recipe = await loadWorkspaceRecipe(recipePath)
     const issues = [
       ...await validateWorkspaceRecipe(recipe, recipePath),
-      ...validateRecipeRuntimePolicy(recipe, options.policy ?? recipePolicy(recipe)),
+      ...validateRecipeRuntimePolicy(recipe, options.policy ?? recipePolicy(recipe, dirname(recipePath)), dirname(recipePath)),
     ]
 
     return {
diff --git a/packages/cli/src/recipe-dry-run.ts b/packages/cli/src/recipe-dry-run.ts
@@ -275,7 +275,7 @@ export async function dryRunRecipe(options: RecipeDryRunOptions, context: Recipe
 
     const issues = [
       ...await validateWorkspaceRecipe(recipe, recipePath),
-      ...validateRecipeRuntimePolicy(recipe, options.policy),
+      ...validateRecipeRuntimePolicy(recipe, options.policy, recipeDirectory),
     ]
 
     if (issues.length > 0) {
@@ -324,7 +324,7 @@ export async function dryRunRecipe(options: RecipeDryRunOptions, context: Recipe
 }
 
 export async function planWorkspaceRecipe(recipe: WorkspaceRecipe, recipeDirectory: string, options: RecipePlanOptions, context: RecipePlanContext): Promise<RecipePlan> {
-  const policy = options.policy ?? recipePolicy(recipe)
+  const policy = options.policy ?? recipePolicy(recipe, recipeDirectory)
   const policyValidation = validateRuntimePolicy(policy)
   const workspaces = recipeDryRunWorkspaces(recipe, recipeDirectory)
   const extraPlugins = recipeDryRunExtraPlugins(recipe, recipeDirectory)
diff --git a/packages/cli/src/recipe-validation.ts b/packages/cli/src/recipe-validation.ts
@@ -1,3 +1,4 @@
+import { readFileSync } from "node:fs"
 import { readFile, stat } from "node:fs/promises"
 import { dirname, join, resolve } from "node:path"
 import { BROWSER_PROBE_CHROMIUM_PROFILE_IDS, RUNTIME_BACKED_FUZZ_SUITE_RUNNER_CAPABILITIES, assertFixtureImportDeterministicIdsSupported, assertWorkspaceRecipeJsonSchema, browserEnvironment, commandArgValue, normalizeRuntimeBackendKind, normalizeRuntimeMountTarget, parseCommandJson, safeArtifactRelativePath, validateBrowserInteractionScript, validateRuntimePolicy, validateSourcePackage, workspaceRecipeRuntimeCollectedArtifacts, type MountSpec, type RuntimeAssetSpec, type RuntimePolicy, type RuntimePreviewSpec, type WorkspaceRecipe, type WorkspaceRecipeDeclaredArtifact, type WorkspaceRecipeDependencyOverlay, type WorkspaceRecipeDistribution, type WorkspaceRecipeDistributionStartupProbe, type WorkspaceRecipeFixtureDatabase, type WorkspaceRecipeFuzzCasePhase, type WorkspaceRecipeMount, type WorkspaceRecipePluginRuntime, type WorkspaceRecipePluginRuntimeHealthProbe, type WorkspaceRecipeProbe, type WorkspaceRecipeRuntimeBackendPackage, type WorkspaceRecipeRuntimeOverlay, type WorkspaceRecipeSiteSeed } from "@automattic/wp-codebox-core"
@@ -513,7 +514,7 @@ export async function validateWorkspaceRecipe(recipe: WorkspaceRecipe, recipePat
   return validateWorkspaceRecipeSemantics(recipe, recipePath)
 }
 
-export function validateRecipeRuntimePolicy(recipe: WorkspaceRecipe, policy: RuntimePolicy | undefined): RecipeValidationIssue[] {
+export function validateRecipeRuntimePolicy(recipe: WorkspaceRecipe, policy: RuntimePolicy | undefined, recipeDirectory?: string): RecipeValidationIssue[] {
   if (!policy) {
     return []
   }
@@ -528,7 +529,7 @@ export function validateRecipeRuntimePolicy(recipe: WorkspaceRecipe, policy: Run
     })
   }
 
-  const requiredCommands = recipePolicy(recipe).commands
+  const requiredCommands = recipePolicy(recipe, recipeDirectory).commands
   for (const command of requiredCommands) {
     if (!policy.commands.includes(command)) {
       issues.push({
@@ -1051,7 +1052,7 @@ function recipeFuzzWorkflowSteps(recipe: WorkspaceRecipe): RecipeWorkflowStepRef
   }))))
 }
 
-export function recipePolicy(recipe: WorkspaceRecipe): RuntimePolicy {
+export function recipePolicy(recipe: WorkspaceRecipe, recipeDirectory?: string): RuntimePolicy {
   const pluginRuntimeCommands = [
     ...(recipe.inputs?.pluginRuntime?.setup ?? []),
     ...(recipe.inputs?.pluginRuntime?.healthProbes ?? []).map(pluginRuntimeHealthProbeStep),
@@ -1091,7 +1092,7 @@ export function recipePolicy(recipe: WorkspaceRecipe): RuntimePolicy {
   // Auto-grant the evaluate capability when a browser-actions step opts into the
   // arbitrary-JS escape hatch by including an evaluate step. Recipe authors opt in
   // by writing the step; direct `run` invocations still control the gate via --policy.
-  if (recipeDeclaredWorkflowSteps(recipe).some(({ step }) => (step.command === "wordpress.browser-actions" || step.command === "wordpress.browser-scenario") && recipeStepUsesEvaluate(step))) {
+  if (recipeDeclaredWorkflowSteps(recipe).some(({ step }) => (step.command === "wordpress.browser-actions" || step.command === "wordpress.browser-scenario") && recipeStepUsesEvaluate(step, recipeDirectory))) {
     commands.push("wordpress.browser-actions.evaluate")
   }
 
@@ -1649,10 +1650,10 @@ async function validateRecipeStepArgs(step: WorkspaceRecipe["workflow"]["steps"]
   if (step.command === "wordpress.browser-actions") {
     const stepsJson = recipeStepArgValue(step.args ?? [], "steps-json")
 
-    if (stepsJson && !stepsJson.startsWith("@")) {
+    if (stepsJson) {
       let parsed: unknown
       try {
-        parsed = JSON.parse(stepsJson)
+        parsed = JSON.parse(stepsJson.startsWith("@") ? await readFile(resolve(recipeDirectory, stepsJson.slice(1)), "utf8") : stepsJson)
       } catch (error) {
         addIssue("invalid-steps-json", `${path}.args`, `wordpress.browser-actions steps-json must be valid JSON: ${error instanceof Error ? error.message : String(error)}`)
         parsed = undefined
@@ -1670,9 +1671,9 @@ async function validateRecipeStepArgs(step: WorkspaceRecipe["workflow"]["steps"]
   if (step.command === "wordpress.browser-scenario") {
     const scenarioJson = recipeStepArgValue(step.args ?? [], "scenario-json")
 
-    if (scenarioJson && !scenarioJson.startsWith("@")) {
+    if (scenarioJson) {
       try {
-        const parsed = JSON.parse(scenarioJson) as unknown
+        const parsed = JSON.parse(scenarioJson.startsWith("@") ? await readFile(resolve(recipeDirectory, scenarioJson.slice(1)), "utf8") : scenarioJson) as unknown
         if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
           addIssue("invalid-scenario-json", `${path}.args`, "wordpress.browser-scenario scenario-json must be a JSON object.")
         } else {
@@ -1702,10 +1703,10 @@ async function validateRecipeStepArgs(step: WorkspaceRecipe["workflow"]["steps"]
     }
 
     const stepsJson = recipeStepArgValue(step.args ?? [], "steps-json")
-    if (stepsJson && !stepsJson.startsWith("@")) {
+    if (stepsJson) {
       let parsed: unknown
       try {
-        parsed = JSON.parse(stepsJson)
+        parsed = JSON.parse(stepsJson.startsWith("@") ? await readFile(resolve(recipeDirectory, stepsJson.slice(1)), "utf8") : stepsJson)
       } catch (error) {
         addIssue("invalid-steps-json", `${path}.args`, `wordpress.browser-scenario steps-json must be valid JSON: ${error instanceof Error ? error.message : String(error)}`)
       }
@@ -1969,31 +1970,37 @@ function recipeBenchWorkloadsUseWpCli(value: unknown): boolean {
   return record.type === "wp-cli" || recipeBenchWorkloadsUseWpCli(record.run)
 }
 
-function recipeStepUsesEvaluate(step: WorkspaceRecipe["workflow"]["steps"][number]): boolean {
+function recipeStepUsesEvaluate(step: WorkspaceRecipe["workflow"]["steps"][number], recipeDirectory?: string): boolean {
   const scenarioRaw = recipeStepArgValue(step.args ?? [], "scenario-json")
-  if (scenarioRaw && !scenarioRaw.startsWith("@")) {
+  if (scenarioRaw) {
     try {
-      const parsed = parseCommandJson(scenarioRaw, "scenario-json") as { steps?: unknown; assertions?: unknown }
+      const parsed = parseCommandJson(recipeJsonArgForPolicy(scenarioRaw, recipeDirectory), "scenario-json") as { steps?: unknown; assertions?: unknown }
       const steps = normalizeBrowserScenarioStepsForValidation(parsed.steps)
       const assertions = normalizeBrowserScenarioAssertionsForValidation(parsed.assertions)
-      return [...steps, ...assertions].some((entry) => entry && typeof entry === "object" && (entry as { kind?: unknown }).kind === "evaluate")
+      if ([...steps, ...assertions].some((entry) => entry && typeof entry === "object" && (entry as { kind?: unknown }).kind === "evaluate")) return true
     } catch {
-      return false
+      // Semantic validation reports malformed or missing file-backed payloads.
     }
   }
 
   const raw = recipeStepArgValue(step.args ?? [], "steps-json")
-  if (!raw || raw.startsWith("@")) {
+  if (!raw) {
     return false
   }
   try {
-    const parsed = parseCommandJson(raw, "steps-json")
+    const parsed = parseCommandJson(recipeJsonArgForPolicy(raw, recipeDirectory), "steps-json")
     return Array.isArray(parsed) && parsed.some((entry) => entry && typeof entry === "object" && (entry as { kind?: unknown }).kind === "evaluate")
   } catch {
     return false
   }
 }
 
+function recipeJsonArgForPolicy(raw: string, recipeDirectory?: string): string {
+  if (!raw.startsWith("@")) return raw
+  if (!recipeDirectory) return ""
+  return readFileSync(resolve(recipeDirectory, raw.slice(1)), "utf8")
+}
+
 function normalizeBrowserScenarioStepsForValidation(value: unknown): unknown[] {
   if (!Array.isArray(value)) {
     return []
diff --git a/packages/runtime-playground/src/browser-actions-runner.ts b/packages/runtime-playground/src/browser-actions-runner.ts
@@ -699,6 +699,8 @@ async function browserActionsRunPlanFromArgs(args: string[], artifactRoot: strin
     capture.add("screenshot")
     capture.add("dom-snapshot")
   }
+  const argumentEnvironment = await browserEnvironmentFromArgs(args)
+  const adaptiveExplorationPlan = browserAdaptiveExplorationFromArgs(args, argumentEnvironment)
   return {
     initialUrl: argValue(args, "url")?.trim(),
     steps: await browserInteractionStepsFromArgs(args),
@@ -707,12 +709,12 @@ async function browserActionsRunPlanFromArgs(args: string[], artifactRoot: strin
     totalTimeoutMs: durationArg(args, "timeout", BROWSER_SCRIPT_DEFAULT_TIMEOUT_MS),
     networkSettleTimeoutMs: durationArg(args, "network-settle-timeout", browserCommandLivenessPolicy().networkSettleTimeoutMs),
     requestedViewport: viewportArg(args, "viewport"),
-    requestedEnvironment: await browserEnvironmentFromArgs(args),
+    requestedEnvironment: adaptiveExplorationPlan.requestedEnvironment,
     authRequest: browserAuthRequest(args),
     storageStateImport: await browserStorageStateImportFromArgs(args, "wordpress.browser-actions", artifactRoot),
     maxDomSnapshotElements: positiveIntegerArg(args, "max-dom-snapshot-elements", 160),
     actionCorpus: browserActionCorpusFromArgs(args),
-    adaptiveExploration: browserAdaptiveExplorationFromArgs(args),
+    adaptiveExploration: adaptiveExplorationPlan.contract,
   }
 }
 
@@ -774,11 +776,23 @@ function browserActionCorpusFromArgs(args: string[]): BrowserActionCorpusContrac
   return browserActionCorpusContract(parsed)
 }
 
-function browserAdaptiveExplorationFromArgs(args: string[]): BrowserAdaptiveExplorationContract | undefined {
+function browserAdaptiveExplorationFromArgs(args: string[], argumentEnvironment: BrowserEnvironment): { contract?: BrowserAdaptiveExplorationContract; requestedEnvironment: BrowserEnvironment } {
   const raw = argValue(args, "adaptive-exploration-json")
-  if (typeof raw !== "string" || raw.trim().length === 0) return undefined
+  if (typeof raw !== "string" || raw.trim().length === 0) return { requestedEnvironment: argumentEnvironment }
   const parsed = JSON.parse(raw) as Record<string, unknown>
-  return browserAdaptiveExplorationContract(parsed)
+  const contract = browserAdaptiveExplorationContract(parsed)
+  if (Object.prototype.hasOwnProperty.call(parsed, "environment")) {
+    if (browserEnvironmentArgsDeclared(args)) {
+      throw new Error("wordpress.browser-actions adaptive-exploration-json environment cannot be combined with outer browser environment arguments")
+    }
+    return { contract, requestedEnvironment: contract.environment }
+  }
+  return { contract, requestedEnvironment: argumentEnvironment }
+}
+
+function browserEnvironmentArgsDeclared(args: string[]): boolean {
+  const names = new Set(["browser-environment-json", "viewport", "device", "user-agent", "permissions", "locale", "timezone", "device-scale-factor", "is-mobile", "has-touch", "geolocation-latitude", "geolocation-longitude", "geolocation-accuracy", "geolocation-permission"])
+  return args.some((arg) => names.has(arg.slice(0, arg.indexOf("="))))
 }
 
 async function captureBrowserActionDomSnapshot({
@@ -1103,7 +1117,7 @@ function browserScenarioCaptures(scenario: BrowserScenarioInput, args: string[])
 
 async function browserScenarioRunPlan(scenario: BrowserScenarioInput, args: string[], url: string, artifactRoot: string, requestedEnvironment: BrowserEnvironment): Promise<BrowserRunPlan> {
   const captures = browserScenarioCaptures(scenario, args)
-  const steps = browserScenarioSteps(scenario, args)
+  const steps = await browserScenarioSteps(scenario, args)
   const assertions = browserScenarioAssertions(scenario)
   const actionSteps = [...steps, ...assertions]
   const requestedViewport = requestedEnvironment.viewport
@@ -1200,8 +1214,8 @@ function browserScenarioActionCaptures(captures: string[]): string[] {
   return selected.length > 0 ? selected : ["steps", "console", "errors", "html", "network", "screenshot", "dom-snapshot"]
 }
 
-function browserScenarioSteps(scenario: BrowserScenarioInput, args: string[]): Array<Record<string, unknown>> {
-  const raw = scenario.steps ?? parseInlineJsonArrayArg(args, "steps-json")
+async function browserScenarioSteps(scenario: BrowserScenarioInput, args: string[]): Promise<Array<Record<string, unknown>>> {
+  const raw = scenario.steps ?? await parseJsonArrayArg(args, "steps-json")
   return (raw ?? []).map((step) => normalizeBrowserScenarioStep(step))
 }
 
@@ -1239,12 +1253,11 @@ function normalizeBrowserScenarioAssertion(assertion: Record<string, unknown>):
   return assertion
 }
 
-function parseInlineJsonArrayArg(args: string[], name: string): Array<Record<string, unknown>> | undefined {
+async function parseJsonArrayArg(args: string[], name: string): Promise<Array<Record<string, unknown>> | undefined> {
   const raw = argValue(args, name)
-  if (!raw || raw.startsWith("@")) {
-    return undefined
-  }
-  const parsed = JSON.parse(raw) as unknown
+  if (!raw) return undefined
+  const text = raw.startsWith("@") ? await readFile(resolveCommandPath(raw.slice(1)), "utf8") : raw
+  const parsed = JSON.parse(text) as unknown
   if (!Array.isArray(parsed)) {
     throw new Error(`wordpress.browser-scenario ${name} must be a JSON array`)
   }
diff --git a/tests/browser-actions-environment.browser.test.ts b/tests/browser-actions-environment.browser.test.ts
diff --git a/tests/recipe-validation-descriptors.test.ts b/tests/recipe-validation-descriptors.test.ts

Original file line number	Diff line number	Diff line change
`@@ -275,7 +275,7 @@ export async function dryRunRecipe(options: RecipeDryRunOptions, context: Recipe`
`275`	`275`
`276`	`276`	`const issues = [`
`277`	`277`	`...await validateWorkspaceRecipe(recipe, recipePath),`
`278`		`- ...validateRecipeRuntimePolicy(recipe, options.policy),`
	`278`	`+ ...validateRecipeRuntimePolicy(recipe, options.policy, recipeDirectory),`
`279`	`279`	`]`
`280`	`280`
`281`	`281`	`if (issues.length > 0) {`
`@@ -324,7 +324,7 @@ export async function dryRunRecipe(options: RecipeDryRunOptions, context: Recipe`
`324`	`324`	`}`
`325`	`325`
`326`	`326`	`export async function planWorkspaceRecipe(recipe: WorkspaceRecipe, recipeDirectory: string, options: RecipePlanOptions, context: RecipePlanContext): Promise<RecipePlan> {`
`327`		`- const policy = options.policy ?? recipePolicy(recipe)`
	`327`	`+ const policy = options.policy ?? recipePolicy(recipe, recipeDirectory)`
`328`	`328`	`const policyValidation = validateRuntimePolicy(policy)`
`329`	`329`	`const workspaces = recipeDryRunWorkspaces(recipe, recipeDirectory)`
`330`	`330`	`const extraPlugins = recipeDryRunExtraPlugins(recipe, recipeDirectory)`