diff --git a/README.md b/README.md index e69098443..3d79980ba 100644 --- a/README.md +++ b/README.md @@ -739,6 +739,8 @@ For synthetic web performance investigations, enable `capture=performance` and u `wordpress.browser-actions` drives the preview with an ordered interaction script so Codebox can prove a plugin still *works* under interaction, not just that it renders. Pass the script as `steps-json=` (inline JSON, or `@` to read it from a file). Each step is a thin, stable mapping over a Playwright locator action — this is not a test-runner DSL. +Action journeys and adaptive exploration can declare their browser context with `browser-environment-json=` (inline or `@`) or the compatible `device`, `viewport`, `device-scale-factor`, `is-mobile`, `has-touch`, `user-agent`, `permissions`, `locale`, `timezone`, and `geolocation-*` arguments. These controls are applied when the Playwright context is created, so mobile/touch emulation and granted, denied, or prompt geolocation remain active for the full journey. Action summaries keep requested configuration, provider-resolved configuration, and browser-observed state separate, with explicit unsupported and inconclusive dimensions. `wordpress.browser-scenario` accepts the same arguments or an `environment` object inside `scenario-json`; probe collection and authored actions share one owned context/page, while multi-actor scenarios create one isolated configured context per actor. + Step kinds: `navigate` (`url`, optional `waitFor=domcontentloaded|load|networkidle`), `click`/`hover` (`selector` or `text`), `fill`/`type` (`selector`, `value`), `press` (`key`, optional `selector`), `drag` (`from` selector, `to` as `{ "selector": ... }` or `{ "x": n, "y": n }`), `select` (`selector`, `value` or `values`), `waitFor` (`selector` or `waitFor=domcontentloaded|load|networkidle|duration|selector:`), `evaluate` (`expression`, optional `assert` to deep-equal the result), `expect` (`selector`, optional `state=visible|hidden|attached|detached|enabled|disabled|checked|unchecked|editable`), and `screenshot` (optional `name` for a named capture). Every step may set its own `timeout=s`; the command also accepts a global `step-timeout=s` (per step) and `timeout=s` (total-script budget). Both are bounded and deterministic — the run stops cleanly on the first failing step, with no silent partial success. The arbitrary-JS `evaluate` step is policy-gated **separately** from the non-JS interaction steps: a script containing `evaluate` requires `wordpress.browser-actions.evaluate` in the runtime policy in addition to `wordpress.browser-actions`. Click/fill/drag/expect and friends never require the extra grant, so a consumer can allow UI driving while still forbidding arbitrary page JS. diff --git a/package.json b/package.json index 25a2263d2..358fbae3a 100644 --- a/package.json +++ b/package.json @@ -181,6 +181,8 @@ "test:generic-primitives": "npm run test:artifact-path-primitives && npm run test:browser-callback-materialization-contracts && npm run test:browser-canonical-preview-origin && npm run test:source-package-compiler-primitives && npm run test:bench-command-step-behavior && npm run test:generic-ability-runtime-run", "test:browser-artifact-session": "tsx tests/browser-artifact-session.test.ts", "test:browser-environment-matrix": "tsx --test tests/browser-environment-matrix.test.ts tests/browser-environment-matrix.browser.test.ts", + "test:browser-actions-environment": "tsx --test tests/browser-actions-environment.browser.test.ts", + "test:browser-recipe-file-payloads-integration": "tsx tests/browser-recipe-file-payloads.integration.test.ts", "test:browser-diagnostic-providers": "tsx tests/browser-diagnostic-providers.test.ts", "test:browser-capture-html-diagnostics-reliability": "tsx tests/browser-capture-html-diagnostics-reliability.test.ts", "test:browser-provider-permissions": "tsx tests/browser-provider-permissions.test.ts", diff --git a/packages/cli/src/agent-sandbox.ts b/packages/cli/src/agent-sandbox.ts index d0e769d48..ef21251ca 100644 --- a/packages/cli/src/agent-sandbox.ts +++ b/packages/cli/src/agent-sandbox.ts @@ -117,7 +117,7 @@ export function agentRuntimeMounts(options: AgentRuntimeProbeOptions): AgentRunt export async function recipeExecutionSpec(step: WorkspaceRecipe["workflow"]["steps"][number], recipeDirectory: string, sandboxWorkspace?: SandboxWorkspaceContract, options: { inputMountPathMap?: readonly InputMountPathMapping[] } = {}): Promise { const originalArgs = step.args ?? [] - const resolvedStep = { ...step, args: rewriteRecipeExecutionArgs(step.command, originalArgs, options.inputMountPathMap) } + const resolvedStep = { ...step, args: rewriteRecipeExecutionArgs(step.command, originalArgs, recipeDirectory, options.inputMountPathMap) } const finish = (spec: ExecutionSpec & { args?: string[] }): ResolvedRecipeExecutionSpec => { // Commands can generate PHP and serialized payloads after their source args // are resolved, so canonicalize the generated execution spec as well. @@ -186,8 +186,8 @@ export async function recipeExecutionSpec(step: WorkspaceRecipe["workflow"]["ste return finish({ command: resolvedStep.command, args: [...(resolvedStep.args ?? []), ...commandDiagnosticsCaptureArgs(resolvedStep.diagnostics)], diagnostics: resolvedStep.diagnostics }) } -function rewriteRecipeExecutionArgs(command: string, args: readonly string[], inputMountPathMap: readonly InputMountPathMapping[] = []): string[] { - const rewritten = rewriteInputMountPathArgs(args, inputMountPathMap) +function rewriteRecipeExecutionArgs(command: string, args: readonly string[], recipeDirectory: string, inputMountPathMap: readonly InputMountPathMapping[] = []): string[] { + const rewritten = rewriteRecipeBrowserPayloadArgs(command, rewriteInputMountPathArgs(args, inputMountPathMap), recipeDirectory) if (command === "wordpress.run-workload") { return rewriteInputMountPathJsonArgs(rewritten, ["workload-json"], inputMountPathMap) } @@ -197,6 +197,22 @@ function rewriteRecipeExecutionArgs(command: string, args: readonly string[], in return rewritten } +function rewriteRecipeBrowserPayloadArgs(command: string, args: readonly string[], recipeDirectory: string): string[] { + const fileBackedArgs = command === "wordpress.browser-actions" + ? new Set(["steps-json", "browser-environment-json"]) + : command === "wordpress.browser-scenario" + ? new Set(["scenario-json", "steps-json", "browser-environment-json"]) + : undefined + if (!fileBackedArgs) return [...args] + return args.map((arg) => { + const separator = arg.indexOf("=") + if (separator < 0 || !fileBackedArgs.has(arg.slice(0, separator))) return arg + const value = arg.slice(separator + 1) + if (!value.startsWith("@")) return arg + return `${arg.slice(0, separator + 1)}@${resolve(recipeDirectory, value.slice(1))}` + }) +} + async function wordpressRunWorkloadExecutionSpec(step: WorkspaceRecipe["workflow"]["steps"][number], recipeDirectory: string): Promise { const args = step.args ?? [] const workloadJson = commandArgValue(args, "workload-json") diff --git a/packages/cli/src/commands/recipe-run.ts b/packages/cli/src/commands/recipe-run.ts index 5b15338da..b7350b272 100644 --- a/packages/cli/src/commands/recipe-run.ts +++ b/packages/cli/src/commands/recipe-run.ts @@ -110,7 +110,7 @@ export async function runRecipe(options: RecipeRunOptions, interruption?: Recipe await artifactPointer.update({ commandStatus: "queued" }) const issues = [ ...await validateWorkspaceRecipe(recipe, recipePath), - ...validateRecipeRuntimePolicy(recipe, options.policy ?? recipePolicy(recipe)), + ...validateRecipeRuntimePolicy(recipe, options.policy ?? recipePolicy(recipe, recipeDirectory), recipeDirectory), ] if (issues.length > 0) { const failure = { @@ -727,7 +727,7 @@ async function validateRecipe(options: RecipeValidateOptions): Promise 0) { @@ -324,7 +324,7 @@ export async function dryRunRecipe(options: RecipeDryRunOptions, context: Recipe } export async function planWorkspaceRecipe(recipe: WorkspaceRecipe, recipeDirectory: string, options: RecipePlanOptions, context: RecipePlanContext): Promise { - const policy = options.policy ?? recipePolicy(recipe) + const policy = options.policy ?? recipePolicy(recipe, recipeDirectory) const policyValidation = validateRuntimePolicy(policy) const workspaces = recipeDryRunWorkspaces(recipe, recipeDirectory) const extraPlugins = recipeDryRunExtraPlugins(recipe, recipeDirectory) diff --git a/packages/cli/src/recipe-validation.ts b/packages/cli/src/recipe-validation.ts index f8e472eab..0da39e00c 100644 --- a/packages/cli/src/recipe-validation.ts +++ b/packages/cli/src/recipe-validation.ts @@ -1,6 +1,7 @@ +import { readFileSync } from "node:fs" import { readFile, stat } from "node:fs/promises" import { dirname, join, resolve } from "node:path" -import { RUNTIME_BACKED_FUZZ_SUITE_RUNNER_CAPABILITIES, assertFixtureImportDeterministicIdsSupported, assertWorkspaceRecipeJsonSchema, commandArgValue, normalizeRuntimeBackendKind, normalizeRuntimeMountTarget, parseCommandJson, safeArtifactRelativePath, validateBrowserInteractionScript, validateRuntimePolicy, validateSourcePackage, workspaceRecipeRuntimeCollectedArtifacts, type MountSpec, type RuntimeAssetSpec, type RuntimePolicy, type RuntimePreviewSpec, type WorkspaceRecipe, type WorkspaceRecipeDeclaredArtifact, type WorkspaceRecipeDependencyOverlay, type WorkspaceRecipeDistribution, type WorkspaceRecipeDistributionStartupProbe, type WorkspaceRecipeFixtureDatabase, type WorkspaceRecipeFuzzCasePhase, type WorkspaceRecipeMount, type WorkspaceRecipePluginRuntime, type WorkspaceRecipePluginRuntimeHealthProbe, type WorkspaceRecipeProbe, type WorkspaceRecipeRuntimeBackendPackage, type WorkspaceRecipeRuntimeOverlay, type WorkspaceRecipeSiteSeed } from "@automattic/wp-codebox-core" +import { BROWSER_PROBE_CHROMIUM_PROFILE_IDS, RUNTIME_BACKED_FUZZ_SUITE_RUNNER_CAPABILITIES, assertFixtureImportDeterministicIdsSupported, assertWorkspaceRecipeJsonSchema, browserEnvironment, commandArgValue, normalizeRuntimeBackendKind, normalizeRuntimeMountTarget, parseCommandJson, safeArtifactRelativePath, validateBrowserInteractionScript, validateRuntimePolicy, validateSourcePackage, workspaceRecipeRuntimeCollectedArtifacts, type MountSpec, type RuntimeAssetSpec, type RuntimePolicy, type RuntimePreviewSpec, type WorkspaceRecipe, type WorkspaceRecipeDeclaredArtifact, type WorkspaceRecipeDependencyOverlay, type WorkspaceRecipeDistribution, type WorkspaceRecipeDistributionStartupProbe, type WorkspaceRecipeFixtureDatabase, type WorkspaceRecipeFuzzCasePhase, type WorkspaceRecipeMount, type WorkspaceRecipePluginRuntime, type WorkspaceRecipePluginRuntimeHealthProbe, type WorkspaceRecipeProbe, type WorkspaceRecipeRuntimeBackendPackage, type WorkspaceRecipeRuntimeOverlay, type WorkspaceRecipeSiteSeed } from "@automattic/wp-codebox-core" import { commandValidationDescriptorFor, effectivePolicyCommandsFor, type CommandArgValidationDescriptor } from "@automattic/wp-codebox-core/contracts" import { composerPackageVendorPath, evaluateRecipeSourcePolicy, isComposerPackageName, pluginTarget, recipeExtraPluginSlug, recipeExtraPluginSource, recipeExtraPluginSourceRoot, recipeExtraPluginSourceSubpath, recipeExtraPlugins, recipeSource, resolveRecipeExtraPluginFile } from "./recipe-sources.js" import { loadConfiguredRuntimeOverlayDescriptors, registeredRuntimeOverlayDescriptors, runtimeOverlayDescriptor, runtimeOverlayTarget } from "./runtime-overlay-registry.js" @@ -513,7 +514,7 @@ export async function validateWorkspaceRecipe(recipe: WorkspaceRecipe, recipePat return validateWorkspaceRecipeSemantics(recipe, recipePath) } -export function validateRecipeRuntimePolicy(recipe: WorkspaceRecipe, policy: RuntimePolicy | undefined): RecipeValidationIssue[] { +export function validateRecipeRuntimePolicy(recipe: WorkspaceRecipe, policy: RuntimePolicy | undefined, recipeDirectory?: string): RecipeValidationIssue[] { if (!policy) { return [] } @@ -528,7 +529,7 @@ export function validateRecipeRuntimePolicy(recipe: WorkspaceRecipe, policy: Run }) } - const requiredCommands = recipePolicy(recipe).commands + const requiredCommands = recipePolicy(recipe, recipeDirectory).commands for (const command of requiredCommands) { if (!policy.commands.includes(command)) { issues.push({ @@ -582,7 +583,7 @@ export async function validateWorkspaceRecipeSemantics(recipe: WorkspaceRecipe, continue } - await validateRecipeStepArgs(step, path, addIssue) + await validateRecipeStepArgs(step, path, addIssue, recipeDirectory) } for (const [index, mount] of (recipe.inputs?.mounts ?? []).entries()) { @@ -693,7 +694,7 @@ export async function validateWorkspaceRecipeSemantics(recipe: WorkspaceRecipe, validateAbsoluteSandboxPath(`${pluginTarget(overlay.consumer, loadAs)}/vendor/${composerPackageVendorPath(overlay.package)}`, `${path}.target`, addIssue) } - await validateRecipePluginRuntime(recipe.inputs?.pluginRuntime, addIssue) + await validateRecipePluginRuntime(recipe.inputs?.pluginRuntime, addIssue, recipeDirectory) for (const [index, fixture] of (recipe.inputs?.fixtureDatabases ?? []).entries()) { const path = `$.inputs.fixtureDatabases[${index}]` @@ -711,7 +712,7 @@ export async function validateWorkspaceRecipeSemantics(recipe: WorkspaceRecipe, addIssue("unsupported-command", `${path}.command`, `Unsupported recipe probe command: ${probe.step.command}`) continue } - await validateRecipeStepArgs(probe.step, path, addIssue) + await validateRecipeStepArgs(probe.step, path, addIssue, recipeDirectory) } for (const [index, name] of (recipe.inputs?.secretEnv ?? []).entries()) { @@ -1051,7 +1052,7 @@ function recipeFuzzWorkflowSteps(recipe: WorkspaceRecipe): RecipeWorkflowStepRef })))) } -export function recipePolicy(recipe: WorkspaceRecipe): RuntimePolicy { +export function recipePolicy(recipe: WorkspaceRecipe, recipeDirectory?: string): RuntimePolicy { const pluginRuntimeCommands = [ ...(recipe.inputs?.pluginRuntime?.setup ?? []), ...(recipe.inputs?.pluginRuntime?.healthProbes ?? []).map(pluginRuntimeHealthProbeStep), @@ -1091,7 +1092,7 @@ export function recipePolicy(recipe: WorkspaceRecipe): RuntimePolicy { // Auto-grant the evaluate capability when a browser-actions step opts into the // arbitrary-JS escape hatch by including an evaluate step. Recipe authors opt in // by writing the step; direct `run` invocations still control the gate via --policy. - if (recipeDeclaredWorkflowSteps(recipe).some(({ step }) => (step.command === "wordpress.browser-actions" || step.command === "wordpress.browser-scenario") && recipeStepUsesEvaluate(step))) { + if (recipeDeclaredWorkflowSteps(recipe).some(({ step }) => (step.command === "wordpress.browser-actions" || step.command === "wordpress.browser-scenario") && recipeStepUsesEvaluate(step, recipeDirectory))) { commands.push("wordpress.browser-actions.evaluate") } @@ -1318,7 +1319,7 @@ export function recipeWpCliCommandFromArgs(args: string[]): string { return recipeStepArgValue(args, "command")?.trim() ?? args.join(" ").trim() } -async function validateRecipePluginRuntime(pluginRuntime: WorkspaceRecipePluginRuntime | undefined, addIssue: (code: string, path: string, message: string) => void): Promise { +async function validateRecipePluginRuntime(pluginRuntime: WorkspaceRecipePluginRuntime | undefined, addIssue: (code: string, path: string, message: string) => void, recipeDirectory: string): Promise { if (!pluginRuntime) { return } @@ -1366,7 +1367,7 @@ async function validateRecipePluginRuntime(pluginRuntime: WorkspaceRecipePluginR addIssue("unsupported-plugin-runtime-setup-command", `${path}.command`, `Unsupported plugin runtime setup command: ${step.command}`) continue } - await validateRecipeStepArgs(step, path, addIssue) + await validateRecipeStepArgs(step, path, addIssue, recipeDirectory) } for (const [index, probe] of (pluginRuntime.healthProbes ?? []).entries()) { @@ -1501,7 +1502,7 @@ export function hasExplicitSiteSeedSelectors(scope: NonNullable Array.isArray(values) && values.length > 0) } -async function validateRecipeStepArgs(step: WorkspaceRecipe["workflow"]["steps"][number], path: string, addIssue: (code: string, path: string, message: string) => void): Promise { +async function validateRecipeStepArgs(step: WorkspaceRecipe["workflow"]["steps"][number], path: string, addIssue: (code: string, path: string, message: string) => void, recipeDirectory: string): Promise { validateRecipeStepDescriptorArgs(step, path, addIssue) if (step.command === "wordpress.run-php" || step.command === "wordpress.phpunit" || step.command === "wordpress.core-phpunit") { @@ -1548,20 +1549,32 @@ async function validateRecipeStepArgs(step: WorkspaceRecipe["workflow"]["steps"] return } - if (step.command === "wordpress.browser-probe") { + if (["wordpress.browser-probe", "wordpress.browser-actions", "wordpress.browser-scenario"].includes(step.command)) { const latitude = recipeStepArgValue(step.args ?? [], "geolocation-latitude") const longitude = recipeStepArgValue(step.args ?? [], "geolocation-longitude") const accuracy = recipeStepArgValue(step.args ?? [], "geolocation-accuracy") const permission = recipeStepArgValue(step.args ?? [], "geolocation-permission") if (Boolean(latitude) !== Boolean(longitude)) { - addIssue("incomplete-geolocation", `${path}.args`, "wordpress.browser-probe geolocation requires both geolocation-latitude and geolocation-longitude.") + addIssue("incomplete-geolocation", `${path}.args`, `${step.command} geolocation requires both geolocation-latitude and geolocation-longitude.`) } if (accuracy && (!latitude || !longitude)) { - addIssue("incomplete-geolocation", `${path}.args`, "wordpress.browser-probe geolocation-accuracy requires geolocation-latitude and geolocation-longitude.") + addIssue("incomplete-geolocation", `${path}.args`, `${step.command} geolocation-accuracy requires geolocation-latitude and geolocation-longitude.`) } if (permission && (!latitude || !longitude)) { - addIssue("incomplete-geolocation", `${path}.args`, "wordpress.browser-probe geolocation-permission requires geolocation-latitude and geolocation-longitude.") + addIssue("incomplete-geolocation", `${path}.args`, `${step.command} geolocation-permission requires geolocation-latitude and geolocation-longitude.`) + } + const environmentJson = recipeStepArgValue(step.args ?? [], "browser-environment-json") + if (environmentJson) { + try { + const rawEnvironment = environmentJson.startsWith("@") ? await readFile(resolve(recipeDirectory, environmentJson.slice(1)), "utf8") : environmentJson + browserEnvironment(JSON.parse(rawEnvironment)) + } catch (error) { + addIssue("invalid-browser-environment", `${path}.args`, `${step.command} browser-environment-json is invalid: ${error instanceof Error ? error.message : String(error)}`) + } } + } + + if (step.command === "wordpress.browser-probe") { for (const assertion of (step.args ?? []).filter((arg) => arg.startsWith("assert=")).map((arg) => arg.slice("assert=".length).trim())) { const rawNormalized = assertion.startsWith("advisory:") ? assertion.slice("advisory:".length).trim() : assertion const frameSeparator = rawNormalized.startsWith("frame:") || rawNormalized.startsWith("frame-url:") ? rawNormalized.indexOf("|") : -1 @@ -1637,10 +1650,10 @@ async function validateRecipeStepArgs(step: WorkspaceRecipe["workflow"]["steps"] if (step.command === "wordpress.browser-actions") { const stepsJson = recipeStepArgValue(step.args ?? [], "steps-json") - if (stepsJson && !stepsJson.startsWith("@")) { + if (stepsJson) { let parsed: unknown try { - parsed = JSON.parse(stepsJson) + parsed = JSON.parse(stepsJson.startsWith("@") ? await readFile(resolve(recipeDirectory, stepsJson.slice(1)), "utf8") : stepsJson) } catch (error) { addIssue("invalid-steps-json", `${path}.args`, `wordpress.browser-actions steps-json must be valid JSON: ${error instanceof Error ? error.message : String(error)}`) parsed = undefined @@ -1658,12 +1671,24 @@ async function validateRecipeStepArgs(step: WorkspaceRecipe["workflow"]["steps"] if (step.command === "wordpress.browser-scenario") { const scenarioJson = recipeStepArgValue(step.args ?? [], "scenario-json") - if (scenarioJson && !scenarioJson.startsWith("@")) { + if (scenarioJson) { try { - const parsed = JSON.parse(scenarioJson) as unknown + const parsed = JSON.parse(scenarioJson.startsWith("@") ? await readFile(resolve(recipeDirectory, scenarioJson.slice(1)), "utf8") : scenarioJson) as unknown if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { addIssue("invalid-scenario-json", `${path}.args`, "wordpress.browser-scenario scenario-json must be a JSON object.") } else { + const profile = (parsed as { profile?: unknown }).profile + if (profile !== undefined && (typeof profile !== "string" || !(BROWSER_PROBE_CHROMIUM_PROFILE_IDS as readonly string[]).includes(profile))) { + addIssue("invalid-profile", `${path}.args`, `wordpress.browser-scenario profile is unsupported: ${String(profile)}`) + } + const environment = (parsed as { environment?: unknown }).environment + if (environment !== undefined) { + try { + browserEnvironment(environment as Parameters[0]) + } catch (error) { + addIssue("invalid-browser-environment", `${path}.args`, `wordpress.browser-scenario environment is invalid: ${error instanceof Error ? error.message : String(error)}`) + } + } const steps = (parsed as { steps?: unknown }).steps if (steps !== undefined) { const result = validateBrowserInteractionScript(normalizeBrowserScenarioStepsForValidation(steps)) @@ -1678,10 +1703,10 @@ async function validateRecipeStepArgs(step: WorkspaceRecipe["workflow"]["steps"] } const stepsJson = recipeStepArgValue(step.args ?? [], "steps-json") - if (stepsJson && !stepsJson.startsWith("@")) { + if (stepsJson) { let parsed: unknown try { - parsed = JSON.parse(stepsJson) + parsed = JSON.parse(stepsJson.startsWith("@") ? await readFile(resolve(recipeDirectory, stepsJson.slice(1)), "utf8") : stepsJson) } catch (error) { addIssue("invalid-steps-json", `${path}.args`, `wordpress.browser-scenario steps-json must be valid JSON: ${error instanceof Error ? error.message : String(error)}`) } @@ -1945,31 +1970,37 @@ function recipeBenchWorkloadsUseWpCli(value: unknown): boolean { return record.type === "wp-cli" || recipeBenchWorkloadsUseWpCli(record.run) } -function recipeStepUsesEvaluate(step: WorkspaceRecipe["workflow"]["steps"][number]): boolean { +function recipeStepUsesEvaluate(step: WorkspaceRecipe["workflow"]["steps"][number], recipeDirectory?: string): boolean { const scenarioRaw = recipeStepArgValue(step.args ?? [], "scenario-json") - if (scenarioRaw && !scenarioRaw.startsWith("@")) { + if (scenarioRaw) { try { - const parsed = parseCommandJson(scenarioRaw, "scenario-json") as { steps?: unknown; assertions?: unknown } + const parsed = parseCommandJson(recipeJsonArgForPolicy(scenarioRaw, recipeDirectory), "scenario-json") as { steps?: unknown; assertions?: unknown } const steps = normalizeBrowserScenarioStepsForValidation(parsed.steps) const assertions = normalizeBrowserScenarioAssertionsForValidation(parsed.assertions) - return [...steps, ...assertions].some((entry) => entry && typeof entry === "object" && (entry as { kind?: unknown }).kind === "evaluate") + if ([...steps, ...assertions].some((entry) => entry && typeof entry === "object" && (entry as { kind?: unknown }).kind === "evaluate")) return true } catch { - return false + // Semantic validation reports malformed or missing file-backed payloads. } } const raw = recipeStepArgValue(step.args ?? [], "steps-json") - if (!raw || raw.startsWith("@")) { + if (!raw) { return false } try { - const parsed = parseCommandJson(raw, "steps-json") + const parsed = parseCommandJson(recipeJsonArgForPolicy(raw, recipeDirectory), "steps-json") return Array.isArray(parsed) && parsed.some((entry) => entry && typeof entry === "object" && (entry as { kind?: unknown }).kind === "evaluate") } catch { return false } } +function recipeJsonArgForPolicy(raw: string, recipeDirectory?: string): string { + if (!raw.startsWith("@")) return raw + if (!recipeDirectory) return "" + return readFileSync(resolve(recipeDirectory, raw.slice(1)), "utf8") +} + function normalizeBrowserScenarioStepsForValidation(value: unknown): unknown[] { if (!Array.isArray(value)) { return [] diff --git a/packages/runtime-core/src/browser-adaptive-exploration.ts b/packages/runtime-core/src/browser-adaptive-exploration.ts index 04b9600fb..a7f5210a4 100644 --- a/packages/runtime-core/src/browser-adaptive-exploration.ts +++ b/packages/runtime-core/src/browser-adaptive-exploration.ts @@ -3,6 +3,7 @@ import { createHash } from "node:crypto" import type { BrowserActionCorpusDescriptor, BrowserInteractionStep, BrowserRandomWalkContext } from "./browser-interaction.js" import { browserAccessibilityContract, type BrowserAccessibilityContract, type BrowserAccessibilityEvidence } from "./browser-accessibility.js" import { isPlainObject, stableJson, stripUndefined } from "./object-utils.js" +import { browserEnvironment, browserEnvironmentDigest, type BrowserEnvironment } from "./browser-environment-matrix.js" export const BROWSER_ADAPTIVE_EXPLORATION_SCHEMA = "wp-codebox/browser-adaptive-exploration/v1" as const export const BROWSER_ADAPTIVE_EXPLORATION_ARTIFACT_SCHEMA = "wp-codebox/browser-adaptive-exploration-artifact/v1" as const @@ -36,6 +37,8 @@ export interface BrowserAdaptiveExplorationContract { failOnFinding: boolean accessibility?: BrowserAccessibilityContract metadata?: Record + environment: BrowserEnvironment + environmentDigest: string } export interface BrowserAdaptiveFrameIdentity { @@ -123,6 +126,8 @@ export interface BrowserAdaptiveFinding { expectedStateDigest?: string actions: BrowserAdaptiveAction[] resetPolicy: BrowserAdaptiveExplorationContract["resetPolicy"] + environment: BrowserEnvironment + environmentDigest: string } } @@ -145,7 +150,7 @@ export interface BrowserAdaptiveExplorationResult { findings: number budgetExhausted?: keyof BrowserAdaptiveExplorationContract["budgets"] | "maxKeyboardActions" | "cancelled" | "frontier" } - replay: { schema: typeof BROWSER_ADAPTIVE_EXPLORATION_SCHEMA; seed: string; startUrl: string; contract: BrowserAdaptiveExplorationContract } + replay: { schema: typeof BROWSER_ADAPTIVE_EXPLORATION_SCHEMA; seed: string; startUrl: string; environment: BrowserEnvironment; environmentDigest: string; contract: BrowserAdaptiveExplorationContract } } export interface BrowserAdaptiveExplorationArtifact { @@ -169,6 +174,7 @@ export function browserAdaptiveExplorationContract(input: Record (BROWSER_ADAPTIVE_ACTION_FAMILIES as readonly unknown[]).includes(value)))] const startUrl = string(input.startUrl ?? input.start_url) ?? (context === "admin" ? "/wp-admin/" : context === "editor" ? "/wp-admin/post-new.php" : "/") + const environment = browserEnvironment(isPlainObject(input.environment) ? input.environment as BrowserEnvironment : {}) return stripUndefined({ schema: BROWSER_ADAPTIVE_EXPLORATION_SCHEMA, context, @@ -203,6 +209,8 @@ export function browserAdaptiveExplorationContract(input: Record string } +export function browserEnvironment(input: BrowserEnvironment): BrowserEnvironment { + validateEnvironment(input) + return canonicalEnvironment(input) +} + +export function browserEnvironmentDigest(input: BrowserEnvironment): string { + return createHash("sha256").update("wp-codebox/browser-environment/v1\n").update(stableJson(browserEnvironment(input))).digest("hex") +} + export function browserEnvironmentMatrix(input: Omit & { limits?: Partial; failOnFinding?: boolean }): BrowserEnvironmentMatrix { if (!safeId(input.id) || !input.seed) throw new Error("Browser environment matrices require a safe non-empty id and seed.") if (!Array.isArray(input.dimensions) || input.dimensions.length === 0) throw new Error("Browser environment matrices require at least one dimension.") @@ -320,9 +331,26 @@ function normalizeLimits(input: Partial | undefi function validateEnvironment(environment: BrowserEnvironment): void { if (!environment || typeof environment !== "object" || Array.isArray(environment)) throw new Error("Browser environment values must be objects.") + const supportedKeys = new Set(["viewport", "device", "userAgent", "permissions", "deviceScaleFactor", "isMobile", "hasTouch", "orientation", "zoom", "colorScheme", "reducedMotion", "forcedColors", "contrast", "locale", "timezone", "networkProfile", "cpuProfile", "online", "clock", "geolocation", "capabilities"]) + const unsupportedKeys = Object.keys(environment).filter((key) => !supportedKeys.has(key)) + if (unsupportedKeys.length > 0) throw new Error(`Browser environment contains unsupported controls: ${unsupportedKeys.sort().join(", ")}.`) if (environment.viewport && (!positiveInteger(environment.viewport.width) || !positiveInteger(environment.viewport.height))) throw new Error("Browser environment viewport width and height must be positive integers.") + for (const key of ["device", "locale", "timezone", "networkProfile", "cpuProfile"] as const) { + if (environment[key] !== undefined && (typeof environment[key] !== "string" || environment[key].length === 0)) throw new Error(`Browser environment ${key} must be a non-empty string.`) + } + if (environment.userAgent !== undefined && (typeof environment.userAgent !== "string" || environment.userAgent.length === 0)) throw new Error("Browser environment userAgent must be a non-empty string.") + if (environment.permissions !== undefined && (!Array.isArray(environment.permissions) || environment.permissions.some((permission) => typeof permission !== "string" || permission.length === 0))) throw new Error("Browser environment permissions must be non-empty strings.") + if (environment.isMobile !== undefined && typeof environment.isMobile !== "boolean") throw new Error("Browser environment isMobile must be boolean.") + if (environment.hasTouch !== undefined && typeof environment.hasTouch !== "boolean") throw new Error("Browser environment hasTouch must be boolean.") if (environment.deviceScaleFactor !== undefined && (!Number.isFinite(environment.deviceScaleFactor) || environment.deviceScaleFactor <= 0)) throw new Error("Browser environment deviceScaleFactor must be positive.") + if (environment.online !== undefined && typeof environment.online !== "boolean") throw new Error("Browser environment online must be boolean.") + if (environment.orientation !== undefined && environment.orientation !== "portrait" && environment.orientation !== "landscape") throw new Error("Browser environment orientation must be portrait or landscape.") + if (environment.colorScheme !== undefined && !["light", "dark", "no-preference"].includes(environment.colorScheme)) throw new Error("Browser environment colorScheme is unsupported.") + if (environment.reducedMotion !== undefined && !["reduce", "no-preference"].includes(environment.reducedMotion)) throw new Error("Browser environment reducedMotion is unsupported.") + if (environment.forcedColors !== undefined && !["active", "none"].includes(environment.forcedColors)) throw new Error("Browser environment forcedColors is unsupported.") + if (environment.contrast !== undefined && !["more", "no-preference"].includes(environment.contrast)) throw new Error("Browser environment contrast is unsupported.") if (environment.zoom !== undefined && (!Number.isFinite(environment.zoom) || environment.zoom < 0.25 || environment.zoom > 5)) throw new Error("Browser environment zoom must be between 0.25 and 5.") + if (environment.clock !== undefined && (!environment.clock || typeof environment.clock !== "object" || (environment.clock.mode !== "realtime" && environment.clock.mode !== "fixed"))) throw new Error("Browser environment clock mode must be realtime or fixed.") if (environment.clock?.mode === "fixed" && (!environment.clock.at || !Number.isFinite(Date.parse(environment.clock.at)))) throw new Error("Fixed browser environment clocks require an ISO-compatible at value.") } diff --git a/packages/runtime-core/src/command-registry.ts b/packages/runtime-core/src/command-registry.ts index 92f3b98af..82d9b46d0 100644 --- a/packages/runtime-core/src/command-registry.ts +++ b/packages/runtime-core/src/command-registry.ts @@ -105,6 +105,22 @@ const snapshotScopingAcceptedArgs: CommandDefinition["acceptedArgs"] = [ const browserActionCaptureValues = ["steps", "actions", "console", "errors", "html", "network", "screenshot", "dom-snapshot"] as const const browserScenarioCaptureValues = ["steps", "actions", "console", "errors", "html", "network", "performance", "memory", "screenshot", "dom-snapshot"] as const +const browserActionEnvironmentAcceptedArgs: CommandDefinition["acceptedArgs"] = [ + { name: "browser-environment-json", description: "Optional browser environment object applied when the action context is created. Supports the public BrowserEnvironment fields and @ JSON.", format: "JSON object or @path" }, + { name: "device", description: "Optional built-in Playwright device profile applied when the action context is created.", format: "Playwright device name, e.g. Pixel 5" }, + { name: "user-agent", description: "Optional browser context user agent override.", format: "string" }, + { name: "permissions", description: "Comma-separated browser permissions granted when the action context is created.", format: "comma-separated permission names" }, + { name: "viewport", description: "Optional viewport applied when the action context is created.", format: "x, e.g. 390x844" }, + { name: "device-scale-factor", description: "Optional positive device scale factor applied when the action context is created.", format: "positive number" }, + { name: "is-mobile", description: "Whether the action context uses authentic mobile browser emulation.", format: "boolean" }, + { name: "has-touch", description: "Whether the action context exposes touch input.", format: "boolean" }, + { name: "locale", description: "Optional browser context locale.", format: "BCP 47 locale, e.g. en-US" }, + { name: "timezone", description: "Optional browser context timezone.", format: "IANA timezone, e.g. America/New_York" }, + { name: "geolocation-latitude", description: "Browser context geolocation latitude.", format: "finite number from -90 to 90" }, + { name: "geolocation-longitude", description: "Browser context geolocation longitude.", format: "finite number from -180 to 180" }, + { name: "geolocation-accuracy", description: "Optional browser context geolocation accuracy in meters.", format: `finite number from 0 to ${BROWSER_GEOLOCATION_MAX_ACCURACY_METERS}` }, + { name: "geolocation-permission", description: "Explicit browser context geolocation permission state. default is normalized to prompt.", format: [...BROWSER_GEOLOCATION_PERMISSION_STATES, ...BROWSER_GEOLOCATION_PERMISSION_ALIASES].join("|") }, +] const editorCaptureValues = ["steps", "console", "errors", "html", "screenshot", "editor-state", "editor-validity"] as const const wordpressAuthArtifactProperties = { auth: { type: "string" }, @@ -144,6 +160,14 @@ const browserActionsValidation: CommandValidationDescriptor = { { name: "step-timeout", kind: "duration", code: "invalid-duration", message: "wordpress.browser-actions step-timeout must look like 500ms or 2s." }, { name: "timeout", kind: "duration", code: "invalid-duration", message: "wordpress.browser-actions timeout must look like 500ms or 2s." }, { name: "capture", kind: "comma-list-enum", values: browserActionCaptureValues, code: "invalid-capture", message: "wordpress.browser-actions capture does not support" }, + { name: "viewport", kind: "viewport", code: "invalid-viewport", message: "wordpress.browser-actions viewport must use x, for example 390x844." }, + { name: "device-scale-factor", kind: "number", minimum: 0.01, maximum: 100, code: "invalid-device-scale-factor", message: "wordpress.browser-actions device-scale-factor must be a finite positive number." }, + { name: "is-mobile", kind: "boolean", code: "invalid-is-mobile", message: "wordpress.browser-actions is-mobile must be true or false." }, + { name: "has-touch", kind: "boolean", code: "invalid-has-touch", message: "wordpress.browser-actions has-touch must be true or false." }, + { name: "geolocation-latitude", kind: "number", minimum: -90, maximum: 90, code: "invalid-geolocation-latitude", message: "wordpress.browser-actions geolocation-latitude must be a finite number from -90 to 90." }, + { name: "geolocation-longitude", kind: "number", minimum: -180, maximum: 180, code: "invalid-geolocation-longitude", message: "wordpress.browser-actions geolocation-longitude must be a finite number from -180 to 180." }, + { name: "geolocation-accuracy", kind: "number", minimum: 0, maximum: BROWSER_GEOLOCATION_MAX_ACCURACY_METERS, code: "invalid-geolocation-accuracy", message: `wordpress.browser-actions geolocation-accuracy must be a finite number from 0 to ${BROWSER_GEOLOCATION_MAX_ACCURACY_METERS}.` }, + { name: "geolocation-permission", kind: "enum", values: [...BROWSER_GEOLOCATION_PERMISSION_STATES, ...BROWSER_GEOLOCATION_PERMISSION_ALIASES], code: "invalid-geolocation-permission", message: "wordpress.browser-actions geolocation-permission must be granted, denied, prompt, or default." }, ], } @@ -155,6 +179,7 @@ const browserScenarioValidation: CommandValidationDescriptor = { { name: "step-timeout", kind: "duration", code: "invalid-duration", message: "wordpress.browser-scenario step-timeout must look like 500ms or 2s." }, { name: "timeout", kind: "duration", code: "invalid-duration", message: "wordpress.browser-scenario timeout must look like 500ms or 2s." }, { name: "capture", kind: "comma-list-enum", values: browserScenarioCaptureValues, code: "invalid-capture", message: "wordpress.browser-scenario capture does not support" }, + ...browserActionsValidation.argRules!.filter((rule) => ["viewport", "device-scale-factor", "is-mobile", "has-touch", "geolocation-latitude", "geolocation-longitude", "geolocation-accuracy", "geolocation-permission"].includes(rule.name)), ], } @@ -1190,6 +1215,7 @@ export const commandRegistry = [ { name: "storage-state", description: "Optional Playwright storageState JSON, or @ to JSON, imported into a fresh browser context. Summaries redact cookie/localStorage values and report only schema/kind, counts, hosts, and diagnostics.", format: "JSON object or @path" }, { name: "capture", description: "Comma-separated artifacts to capture after interactions.", format: "steps,console,errors,html,network,screenshot,dom-snapshot" }, { name: "max-dom-snapshot-elements", description: "Maximum visible elements captured in each screenshot sidecar DOM/style snapshot; defaults to 160.", format: "positive integer" }, + ...browserActionEnvironmentAcceptedArgs, ], outputShape: "JSON summary plus files/browser/steps.jsonl, action-summary.json (with assertions pass/fail), optional action-corpus.json or adaptive-exploration.json replay artifacts, named screenshots, sidecar DOM/style snapshots, and optional console/errors/network/html/screenshot artifacts.", policyRequirement: "Runtime policy commands must include wordpress.browser-actions. The evaluate step additionally requires wordpress.browser-actions.evaluate.", @@ -1201,7 +1227,7 @@ export const commandRegistry = [ id: "wordpress.browser-scenario", description: "Run a declarative browser evidence scenario by composing browser-probe and browser-actions artifacts behind one normalized scenario summary.", acceptedArgs: [ - { name: "scenario-json", description: "Declarative scenario object with url, profile, captures, observers, steps, assertions, viewport, and timeout settings. Supports inline JSON or @.", format: "JSON object" }, + { name: "scenario-json", description: "Declarative scenario object with url, profile, environment, captures, observers, steps, assertions, viewport, and timeout settings. Supports inline JSON or @.", format: "JSON object" }, { name: "url", description: "Preview path or absolute URL to visit when scenario-json is omitted or does not include url.", format: "path or URL" }, { name: "steps-json", description: "Optional browser interaction steps when scenario-json.steps is omitted.", format: "JSON array" }, { name: "capture", description: "Comma-separated artifacts to capture.", format: "steps,console,errors,html,network,performance,memory,screenshot,dom-snapshot" }, @@ -1213,8 +1239,10 @@ export const commandRegistry = [ { name: "auth-user-id", description: "WordPress user ID used with auth=wordpress-admin; defaults to 1.", format: "positive integer" }, { name: "step-timeout", description: "Per-step timeout applied to action steps.", format: "duration, e.g. 5s or 500ms" }, { name: "timeout", description: "Total action timeout bounding the interaction run.", format: "duration, e.g. 30s or 1500ms" }, + { name: "storage-state", description: "Optional Playwright storageState JSON, or @ to JSON, shared by scenario probe collection and authored actions.", format: "JSON object or @path" }, + ...browserActionEnvironmentAcceptedArgs.filter((arg) => !["device", "viewport", "locale"].includes(arg.name)), ], - outputShape: "JSON scenario summary with requested/effective browser metadata and files/browser/scenario-summary.json, preserving lower-level browser-probe and browser-actions summaries when used.", + outputShape: "JSON scenario summary with requested, provider-resolved, and browser-observed environment metadata plus files/browser/scenario-summary.json, preserving lower-level browser-probe and browser-actions summaries when used.", policyRequirement: "Runtime policy commands must include wordpress.browser-scenario. Scenarios using evaluate steps additionally require wordpress.browser-actions.evaluate.", validation: browserScenarioValidation, recipe: true, diff --git a/packages/runtime-playground/src/browser-actions-runner.ts b/packages/runtime-playground/src/browser-actions-runner.ts index d48e84462..dfb888b48 100644 --- a/packages/runtime-playground/src/browser-actions-runner.ts +++ b/packages/runtime-playground/src/browser-actions-runner.ts @@ -1,5 +1,5 @@ import { readFile, writeFile } from "node:fs/promises" -import { BROWSER_ACTION_CORPUS_SCHEMA, BROWSER_ADAPTIVE_EXPLORATION_SCHEMA, BROWSER_MULTI_ACTOR_SCENARIO_SCHEMA, BROWSER_TOOL_VERIFIER_RESULT_SCHEMA, HostToolRegistry, assertRuntimeCommandAllowed, browserActionCorpusArtifact, browserActionCorpusContract, browserAdaptiveExplorationContract, browserInteractionScriptUsesEvaluate, browserToolVerifierInputSummary, createHostToolRegistry, executeHostTool, resolveCommandPath, validateBrowserInteractionScript, type BrowserActionCorpusArtifact, type BrowserActionCorpusContract, type BrowserAdaptiveExplorationArtifact, type BrowserAdaptiveExplorationContract, type BrowserInteractionStep, type BrowserMultiActorScenario, type BrowserToolVerifierResult, type ExecutionSpec, type HostToolDefinition, type JsonValue, type RuntimeCreateSpec } from "@automattic/wp-codebox-core" +import { BROWSER_ACTION_CORPUS_SCHEMA, BROWSER_ADAPTIVE_EXPLORATION_SCHEMA, BROWSER_MULTI_ACTOR_SCENARIO_SCHEMA, BROWSER_PROBE_PROFILES, BROWSER_TOOL_VERIFIER_RESULT_SCHEMA, HostToolRegistry, assertRuntimeCommandAllowed, browserActionCorpusArtifact, browserActionCorpusContract, browserAdaptiveExplorationContract, browserEnvironment, browserEnvironmentDigest, browserGeolocation, browserInteractionScriptUsesEvaluate, browserToolVerifierInputSummary, createHostToolRegistry, executeHostTool, resolveCommandPath, validateBrowserInteractionScript, type BrowserActionCorpusArtifact, type BrowserActionCorpusContract, type BrowserAdaptiveExplorationArtifact, type BrowserAdaptiveExplorationContract, type BrowserEnvironment, type BrowserGeolocationPermissionState, type BrowserInteractionStep, type BrowserMultiActorScenario, type BrowserToolVerifierResult, type ExecutionSpec, type HostToolDefinition, type JsonValue, type RuntimeCreateSpec } from "@automattic/wp-codebox-core" import { now, sha256 } from "@automattic/wp-codebox-core/internals" import { browserInteractionStepsFromArgs, browserStepTimeoutMs, durationStringMs, sanitizeScreenshotName } from "./browser-actions.js" import { BrowserArtifactSession } from "./browser-artifact-session.js" @@ -17,13 +17,14 @@ import { BROWSER_PROBE_STATE_INIT_SCRIPT, browserProbeReplayability, browserProb import { runBrowserProbeCommand, type BrowserProbeRunPlan } from "./browser-probe-runner.js" import { browserActionTargetUrls, browserAuthRequest, browserProbeWaterfallArtifact, browserProbeWebSocketArtifact, browserProbeWebSocketSummary, browserRedirectDiagnosticsArtifact, browserRequestCoverageArtifact, browserStorageStateAuthSummary, browserStorageStateImportFromArgs, browserWordPressDiagnosticsArtifact, createBrowserProbeProgressTracker, fileSha256, installBrowserWordPressDiagnostics, installWordPressAdminAuthCookies, livenessRemainingWallTimeMs, normalizeBrowserProbeScriptCheckpoint, type BrowserCommandProgressEvent, type BrowserStorageStateImport } from "./browser-probe-support.js" import { positiveIntegerArg } from "./command-args.js" -import { argValue, commaListArg, durationArg, viewportArg } from "./commands.js" +import { argValue, commaListArg, durationArg, strictBooleanArg, viewportArg } from "./commands.js" import type { PlaygroundRunResponse } from "./playground-command-errors.js" import type { PlaygroundCliServer } from "./preview-server.js" import type { Page } from "playwright" import { discoverBrowserActionCorpusDescriptors } from "./browser-action-discovery.js" import { exploreAdaptiveBrowserStateMachine } from "./browser-adaptive-explorer.js" import { createBrowserAccessibilityCollector } from "./browser-accessibility-collector.js" +import { browserEnvironmentCell, createPlaywrightBrowserEnvironmentContext, observePlaywrightBrowserEnvironment, resolvePlaywrightBrowserEnvironment, type PlaywrightBrowserEnvironmentSession } from "./browser-environment-matrix.js" export { discoverBrowserActionCorpusDescriptors } from "./browser-action-discovery.js" @@ -42,11 +43,13 @@ export interface BrowserActionsRunPlan { totalTimeoutMs: number networkSettleTimeoutMs: number requestedViewport?: { width: number; height: number } + requestedEnvironment?: BrowserEnvironment authRequest?: { userId: number } storageStateImport?: BrowserStorageStateImport maxDomSnapshotElements: number actionCorpus?: BrowserActionCorpusContract adaptiveExploration?: BrowserAdaptiveExplorationContract + reusePage?: boolean } interface BrowserRunPlan { @@ -64,6 +67,7 @@ export async function runBrowserActionsCommand({ server, spec, onProgress, + session, }: { artifactRoot: string plan?: BrowserActionsRunPlan @@ -72,6 +76,7 @@ export async function runBrowserActionsCommand({ server: PlaygroundCliServer spec: ExecutionSpec onProgress?: (event: BrowserCommandProgressEvent) => void + session?: PlaywrightBrowserEnvironmentSession }): Promise<{ artifact: BrowserArtifact; output: string }> { const args = spec.args ?? [] const runPlan = plan ?? await browserActionsRunPlanFromArgs(args, artifactRoot) @@ -84,7 +89,7 @@ export async function runBrowserActionsCommand({ throw new Error("wordpress.browser-actions requires steps-json=, url=, or adaptive-exploration-json=") } - if (initialUrl && steps[0]?.kind !== "navigate") { + if (initialUrl && !runPlan.reusePage && steps[0]?.kind !== "navigate") { steps.unshift({ kind: "navigate", url: initialUrl }) } @@ -106,6 +111,7 @@ export async function runBrowserActionsCommand({ const totalTimeoutMs = runPlan.totalTimeoutMs const livenessPolicy = browserCommandLivenessPolicy({ wallTimeoutMs: totalTimeoutMs, networkSettleTimeoutMs: runPlan.networkSettleTimeoutMs }) const requestedViewport = runPlan.requestedViewport + const requestedEnvironment = browserEnvironment({ ...(requestedViewport ? { viewport: requestedViewport } : {}), ...(runPlan.requestedEnvironment ?? {}) }) const authRequest = runPlan.authRequest const storageStateImport = runPlan.storageStateImport if (authRequest && storageStateImport) { @@ -124,7 +130,7 @@ export async function runBrowserActionsCommand({ const startedAt = now() const startedAtMs = Date.now() const progress = createBrowserProbeProgressTracker(startedAt, 0) - const browser = await launchChromiumBrowser() + const browser = session?.browser ?? await launchChromiumBrowser() const topology = browserPreviewTopology(args, runtimeSpec, server.serverUrl, server.previewProxyDiagnostics?.targetOrigin) const { preview, networkPolicy } = topology let requestedUrl = initialUrl ? topology.resolveUrl(initialUrl) : preview.effectiveOrigin @@ -142,20 +148,34 @@ export async function runBrowserActionsCommand({ let actionCorpusSummary: BrowserArtifact["summary"]["actionCorpus"] | undefined let adaptiveExplorationArtifact: BrowserAdaptiveExplorationArtifact | undefined let adaptiveExplorationSummary: BrowserArtifact["summary"]["adaptiveExploration"] | undefined + let environmentRuntime: Awaited> | undefined + let environmentEvidence: BrowserArtifact["summary"]["environment"] | undefined + let resolvedEnvironment: Awaited> | undefined + let activePage: Page | undefined try { const previewReadinessError = browserPreviewReadinessError(preview) if (previewReadinessError) { throw previewReadinessError } - const context = browserPreviewNeedsContextRouting(networkPolicy) || !!storageStateImport ? await browser.newContext({ - ...topology.contextOptions(), - ...(storageStateImport ? { storageState: storageStateImport.storageState } : {}), - }) : null - if (context) { + resolvedEnvironment = session?.resolved ?? await resolvePlaywrightBrowserEnvironment(browserEnvironmentCell(requestedEnvironment), browser) + const unsupportedEnvironment = resolvedEnvironment.capabilities.filter(({ fidelity }) => fidelity === "unsupported").map(({ id }) => id) + environmentEvidence = { requested: requestedEnvironment, resolved: resolvedEnvironment.effective, provider: resolvedEnvironment.provider, capabilities: resolvedEnvironment.capabilities, unsupported: unsupportedEnvironment, inconclusive: [] } + if (unsupportedEnvironment.length > 0) { + throw new Error(`wordpress.browser-actions browser environment is unsupported: ${unsupportedEnvironment.join(", ")}`) + } + const needsEnvironmentContext = Object.keys(requestedEnvironment).length > 0 || browserPreviewNeedsContextRouting(networkPolicy) || !!storageStateImport + environmentRuntime = session?.runtime ?? (needsEnvironmentContext ? await createPlaywrightBrowserEnvironmentContext(browser, resolvedEnvironment, { + contextOptions: { + ...topology.contextOptions(), + ...(storageStateImport ? { storageState: storageStateImport.storageState } : {}), + }, + }) : undefined) + const context = environmentRuntime?.context ?? null + if (context && !session) { await routeBrowserPreviewContextNetwork(context, networkPolicy, topology.origins.localProxyOrigin) } - const page = context ? await context.newPage() : await browser.newPage() + const page = activePage = environmentRuntime?.page ?? await browser.newPage() if (onProgress) { await page.exposeFunction("__wpCodeboxProbeCheckpointEvent", (checkpoint: unknown) => { const normalized = normalizeBrowserProbeScriptCheckpoint(checkpoint) @@ -173,9 +193,6 @@ export async function runBrowserActionsCommand({ if (authRequest) { authSummary = await installWordPressAdminAuthCookies({ command: "wordpress.browser-actions", cookieUrls: topology.authCookieUrls(browserActionTargetUrls(steps, preview.effectiveOrigin, requestedUrl)), page, runPlaygroundCommand, runtimeSpec, server, userId: authRequest.userId }) } - if (requestedViewport) { - await page.setViewportSize(requestedViewport) - } wordpressDiagnosticsReady = await installBrowserWordPressDiagnostics(runPlaygroundCommand, server) viewport = await browserProbeViewport(page) attachBrowserCaptureListeners({ @@ -234,6 +251,8 @@ export async function runBrowserActionsCommand({ if (runPlan.adaptiveExploration) { const adaptiveContract = { ...runPlan.adaptiveExploration, + environment: requestedEnvironment, + environmentDigest: browserEnvironmentDigest(requestedEnvironment), budgets: { ...runPlan.adaptiveExploration.budgets, maxDurationMs: Math.min(runPlan.adaptiveExploration.budgets.maxDurationMs, livenessRemainingWallTimeMs(startedAtMs, totalTimeoutMs)), @@ -419,7 +438,11 @@ export async function runBrowserActionsCommand({ errors.push(serializeBrowserError("probe-error", error)) } finally { await settleBrowserNetworkTasks(networkTasks, livenessPolicy.networkSettleTimeoutMs) - await browser.close() + if (activePage && resolvedEnvironment) environmentEvidence = await observePlaywrightBrowserEnvironment(activePage, requestedEnvironment, resolvedEnvironment).catch(() => environmentEvidence) + if (!session) { + await environmentRuntime?.close().catch(() => undefined) + await browser.close() + } if (capture.has("steps")) { await artifactSession.writeJsonLines("steps", "steps.jsonl", stepRecords) } @@ -510,6 +533,7 @@ export async function runBrowserActionsCommand({ replayability: browserProbeReplayability(capture), screenshot: capture.has("screenshot"), auth: authSummary, + environment: environmentEvidence, viewport, }, } @@ -541,6 +565,7 @@ export async function runBrowserActionsCommand({ ...(actionCorpusArtifact ? { actionCorpus: actionCorpusArtifact.plan } : {}), ...(adaptiveExplorationArtifact ? { adaptiveExploration: adaptiveExplorationArtifact.result } : {}), viewport, + environment: environmentEvidence, summary: artifact.summary, }) } @@ -674,6 +699,8 @@ async function browserActionsRunPlanFromArgs(args: string[], artifactRoot: strin capture.add("screenshot") capture.add("dom-snapshot") } + const argumentEnvironment = await browserEnvironmentFromArgs(args) + const adaptiveExplorationPlan = browserAdaptiveExplorationFromArgs(args, argumentEnvironment) return { initialUrl: argValue(args, "url")?.trim(), steps: await browserInteractionStepsFromArgs(args), @@ -682,12 +709,64 @@ async function browserActionsRunPlanFromArgs(args: string[], artifactRoot: strin totalTimeoutMs: durationArg(args, "timeout", BROWSER_SCRIPT_DEFAULT_TIMEOUT_MS), networkSettleTimeoutMs: durationArg(args, "network-settle-timeout", browserCommandLivenessPolicy().networkSettleTimeoutMs), requestedViewport: viewportArg(args, "viewport"), + requestedEnvironment: adaptiveExplorationPlan.requestedEnvironment, authRequest: browserAuthRequest(args), storageStateImport: await browserStorageStateImportFromArgs(args, "wordpress.browser-actions", artifactRoot), maxDomSnapshotElements: positiveIntegerArg(args, "max-dom-snapshot-elements", 160), actionCorpus: browserActionCorpusFromArgs(args), - adaptiveExploration: browserAdaptiveExplorationFromArgs(args), + adaptiveExploration: adaptiveExplorationPlan.contract, + } +} + +async function browserEnvironmentFromArgs(args: string[]): Promise { + const raw = argValue(args, "browser-environment-json") + let declared: BrowserEnvironment = {} + if (raw) { + const text = raw.startsWith("@") ? await readFile(resolveCommandPath(raw.slice(1)), "utf8") : raw + declared = JSON.parse(text) as BrowserEnvironment } + return browserEnvironmentFromValues(args, declared) +} + +function browserEnvironmentFromValues(args: string[], declared: BrowserEnvironment): BrowserEnvironment { + const viewport = viewportArg(args, "viewport") + const device = argValue(args, "device")?.trim() + const userAgent = argValue(args, "user-agent")?.trim() + const permissions = commaListArg(args, "permissions") + const locale = argValue(args, "locale")?.trim() + const timezone = argValue(args, "timezone")?.trim() + const deviceScaleFactorRaw = argValue(args, "device-scale-factor")?.trim() + const isMobile = argValue(args, "is-mobile") === undefined ? undefined : strictBooleanArg(args, "is-mobile", false) + const hasTouch = argValue(args, "has-touch") === undefined ? undefined : strictBooleanArg(args, "has-touch", false) + const geolocation = browserActionGeolocation(args) + return browserEnvironment({ + ...declared, + ...(viewport ? { viewport } : {}), + ...(device ? { device } : {}), + ...(userAgent ? { userAgent } : {}), + ...(permissions.length > 0 ? { permissions } : {}), + ...(deviceScaleFactorRaw ? { deviceScaleFactor: Number(deviceScaleFactorRaw) } : {}), + ...(isMobile !== undefined ? { isMobile } : {}), + ...(hasTouch !== undefined ? { hasTouch } : {}), + ...(locale ? { locale } : {}), + ...(timezone ? { timezone } : {}), + ...(geolocation ? { geolocation } : {}), + }) +} + +function browserActionGeolocation(args: string[]): BrowserEnvironment["geolocation"] { + const latitudeRaw = argValue(args, "geolocation-latitude")?.trim() + const longitudeRaw = argValue(args, "geolocation-longitude")?.trim() + const accuracyRaw = argValue(args, "geolocation-accuracy")?.trim() + const permissionRaw = argValue(args, "geolocation-permission")?.trim() + if (!latitudeRaw && !longitudeRaw && !accuracyRaw && !permissionRaw) return undefined + if (!latitudeRaw || !longitudeRaw) throw new Error("wordpress.browser-actions geolocation requires both geolocation-latitude and geolocation-longitude.") + return browserGeolocation({ + latitude: Number(latitudeRaw), + longitude: Number(longitudeRaw), + ...(accuracyRaw ? { accuracy: Number(accuracyRaw) } : {}), + permission: (permissionRaw ?? "prompt") as BrowserGeolocationPermissionState | "default", + }) } function browserActionCorpusFromArgs(args: string[]): BrowserActionCorpusContract | undefined { @@ -697,11 +776,23 @@ function browserActionCorpusFromArgs(args: string[]): BrowserActionCorpusContrac return browserActionCorpusContract(parsed) } -function browserAdaptiveExplorationFromArgs(args: string[]): BrowserAdaptiveExplorationContract | undefined { +function browserAdaptiveExplorationFromArgs(args: string[], argumentEnvironment: BrowserEnvironment): { contract?: BrowserAdaptiveExplorationContract; requestedEnvironment: BrowserEnvironment } { const raw = argValue(args, "adaptive-exploration-json") - if (typeof raw !== "string" || raw.trim().length === 0) return undefined + if (typeof raw !== "string" || raw.trim().length === 0) return { requestedEnvironment: argumentEnvironment } const parsed = JSON.parse(raw) as Record - return browserAdaptiveExplorationContract(parsed) + const contract = browserAdaptiveExplorationContract(parsed) + if (Object.prototype.hasOwnProperty.call(parsed, "environment")) { + if (browserEnvironmentArgsDeclared(args)) { + throw new Error("wordpress.browser-actions adaptive-exploration-json environment cannot be combined with outer browser environment arguments") + } + return { contract, requestedEnvironment: contract.environment } + } + return { contract, requestedEnvironment: argumentEnvironment } +} + +function browserEnvironmentArgsDeclared(args: string[]): boolean { + const names = new Set(["browser-environment-json", "viewport", "device", "user-agent", "permissions", "locale", "timezone", "device-scale-factor", "is-mobile", "has-touch", "geolocation-latitude", "geolocation-longitude", "geolocation-accuracy", "geolocation-permission"]) + return args.some((arg) => names.has(arg.slice(0, arg.indexOf("=")))) } async function captureBrowserActionDomSnapshot({ @@ -826,6 +917,7 @@ interface BrowserScenarioInput { assertions?: Array> viewport?: string device?: string + environment?: BrowserEnvironment locale?: string auth?: string authUserId?: string | number @@ -856,7 +948,8 @@ export async function runBrowserScenarioCommand({ }): Promise<{ artifact: BrowserArtifact; output: string }> { const args = spec.args ?? [] const scenario = await browserScenarioFromArgs(args) - const multiActorScenario = browserMultiActorScenario(scenario, args) + const requestedEnvironment = await browserScenarioEnvironment(scenario, args) + const multiActorScenario = browserMultiActorScenario(scenario, args, requestedEnvironment) if (multiActorScenario) { return runBrowserMultiActorScenarioCommand({ artifactRoot, scenario: multiActorScenario, runtimeSpec, runPlaygroundCommand, server }) } @@ -865,34 +958,56 @@ export async function runBrowserScenarioCommand({ throw new Error("wordpress.browser-scenario requires url= or scenario-json.url") } - const runPlan = browserScenarioRunPlan(scenario, args, url) + const runPlan = await browserScenarioRunPlan(scenario, args, url, artifactRoot, requestedEnvironment) const startedAt = now() const artifactSession = new BrowserArtifactSession(artifactRoot, "files/browser", { source: "wordpress.browser-scenario", operation: "browser-scenario" }) let probeResult: Awaited> | undefined let actionsResult: Awaited> | undefined let pendingError: Error | undefined + let scenarioSession: PlaywrightBrowserEnvironmentSession | undefined + let scenarioBrowser: Awaited> | undefined - if (runPlan.probe) { - try { - probeResult = await runBrowserProbeCommand({ artifactRoot, plan: runPlan.probe, runtimeSpec, runPlaygroundCommand, server, spec: { ...spec, command: "wordpress.browser-probe", args } }) - } catch (error) { - if (isBrowserCommandArtifactError(error) && error.artifact.artifactType === "probe") { - probeResult = { artifact: error.artifact, output: "" } + try { + if (runPlan.probe && runPlan.actions) { + const browser = scenarioBrowser = await launchChromiumBrowser() + const resolved = await resolvePlaywrightBrowserEnvironment(browserEnvironmentCell(requestedEnvironment), browser) + const unsupported = resolved.capabilities.filter(({ fidelity }) => fidelity === "unsupported").map(({ id }) => id) + if (unsupported.length > 0) { + throw new Error(`wordpress.browser-scenario browser environment is unsupported: ${unsupported.join(", ")}`) } - pendingError = error instanceof Error ? error : new Error(String(error)) + const runtime = await createPlaywrightBrowserEnvironmentContext(browser, resolved, { + ...(runPlan.actions.storageStateImport ? { contextOptions: { storageState: runPlan.actions.storageStateImport.storageState } } : {}), + }) + const topology = browserPreviewTopology(args, runtimeSpec, server.serverUrl) + if (browserPreviewNeedsContextRouting(topology.networkPolicy)) await routeBrowserPreviewContextNetwork(runtime.context, topology.networkPolicy, topology.preview.effectiveOrigin) + scenarioSession = { browser, requested: requestedEnvironment, resolved, runtime } } - } - if (!pendingError && runPlan.actions) { - try { - actionsResult = await runBrowserActionsCommand({ artifactRoot, plan: runPlan.actions, runtimeSpec, runPlaygroundCommand, server, spec: { ...spec, command: "wordpress.browser-actions", args } }) - } catch (error) { - if (isBrowserCommandArtifactError(error) && error.artifact.artifactType === "actions") { - actionsResult = { artifact: error.artifact, output: "" } + if (runPlan.probe) { + try { + probeResult = await runBrowserProbeCommand({ artifactRoot, plan: runPlan.probe, runtimeSpec, runPlaygroundCommand, server, spec: { ...spec, command: "wordpress.browser-probe", args }, session: scenarioSession }) + } catch (error) { + if (isBrowserCommandArtifactError(error) && error.artifact.artifactType === "probe") { + probeResult = { artifact: error.artifact, output: "" } + } + pendingError = error instanceof Error ? error : new Error(String(error)) + } + } + + if (!pendingError && runPlan.actions) { + try { + actionsResult = await runBrowserActionsCommand({ artifactRoot, plan: { ...runPlan.actions, reusePage: Boolean(scenarioSession) }, runtimeSpec, runPlaygroundCommand, server, spec: { ...spec, command: "wordpress.browser-actions", args }, session: scenarioSession }) + } catch (error) { + if (isBrowserCommandArtifactError(error) && error.artifact.artifactType === "actions") { + actionsResult = { artifact: error.artifact, output: "" } + } + pendingError = error instanceof Error ? error : new Error(String(error)) } - pendingError = error instanceof Error ? error : new Error(String(error)) } + } finally { + await scenarioSession?.runtime.close().catch(() => undefined) + await scenarioBrowser?.close().catch(() => undefined) } const primaryArtifact = actionsResult?.artifact ?? probeResult?.artifact @@ -967,18 +1082,19 @@ export async function runBrowserScenarioCommand({ async function browserScenarioFromArgs(args: string[]): Promise { const raw = argValue(args, "scenario-json") - if (!raw) { - return {} - } - const text = raw.startsWith("@") ? await readFile(resolveCommandPath(raw.slice(1)), "utf8") : raw - const parsed = JSON.parse(text) as unknown - if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { - throw new Error("wordpress.browser-scenario scenario-json must be a JSON object") + let scenario: BrowserScenarioInput = {} + if (raw) { + const text = raw.startsWith("@") ? await readFile(resolveCommandPath(raw.slice(1)), "utf8") : raw + const parsed = JSON.parse(text) as unknown + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("wordpress.browser-scenario scenario-json must be a JSON object") + } + scenario = parsed as BrowserScenarioInput } - return parsed as BrowserScenarioInput + return scenario } -function browserMultiActorScenario(scenario: BrowserScenarioInput, args: string[]): (BrowserMultiActorScenario & { url: string; captures?: string[]; stepTimeoutMs?: number }) | undefined { +function browserMultiActorScenario(scenario: BrowserScenarioInput, args: string[], environment: BrowserEnvironment): (BrowserMultiActorScenario & { url: string; captures?: string[]; stepTimeoutMs?: number; environment: BrowserEnvironment; browserArgs: string[] }) | undefined { if (!scenario.actors) return undefined const url = scenario.url?.trim() || argValue(args, "url")?.trim() if (!url) throw new Error("Multi-actor wordpress.browser-scenario requires scenario-json.url or url=") @@ -988,7 +1104,9 @@ function browserMultiActorScenario(scenario: BrowserScenarioInput, args: string[ } if (scenario.barriers && (!Array.isArray(scenario.barriers) || !scenario.barriers.every((barrier) => barrier && typeof barrier.name === "string" && Array.isArray(barrier.actors)))) throw new Error("Multi-actor browser scenario barriers must have a name and actor list") if (scenario.requestGates && (!Array.isArray(scenario.requestGates) || !scenario.requestGates.every((gate) => gate && typeof gate.name === "string" && typeof gate.actor === "string" && typeof gate.url === "string" && (gate.occurrence === undefined || (Number.isInteger(gate.occurrence) && gate.occurrence > 0))))) throw new Error("Multi-actor browser scenario request gates must have a name, actor, URL, and positive occurrence") - return { schema: scenario.schema, seed: scenario.seed, actors: scenario.actors, actions: scenario.actions, ...(scenario.barriers ? { barriers: scenario.barriers } : {}), ...(scenario.requestGates ? { requestGates: scenario.requestGates } : {}), ...(typeof scenario.timeout === "string" ? { timeoutMs: durationStringMs(scenario.timeout) } : {}), url, captures: browserScenarioCaptures(scenario, args), stepTimeoutMs: durationStringMs(scenario.stepTimeout ?? argValue(args, "step-timeout")) || BROWSER_STEP_DEFAULT_TIMEOUT_MS } + if (scenario.prePageScript || argValue(args, "pre-page-script") || scenario.observers || scenario.auth || argValue(args, "auth") || argValue(args, "auth-user-id") || argValue(args, "storage-state")) throw new Error("Multi-actor wordpress.browser-scenario does not support pre-page scripts, observers, shared auth, or storage-state; each actor uses its declared user session.") + const browserArgs = args.filter((arg) => /^(route-host|allow-host|block-host|record-external|preview-mode|network-policy)=/.test(arg)) + return { schema: scenario.schema, seed: scenario.seed, actors: scenario.actors, actions: scenario.actions, ...(scenario.barriers ? { barriers: scenario.barriers } : {}), ...(scenario.requestGates ? { requestGates: scenario.requestGates } : {}), ...(typeof scenario.timeout === "string" ? { timeoutMs: durationStringMs(scenario.timeout) } : {}), url, captures: browserScenarioCaptures(scenario, args), stepTimeoutMs: durationStringMs(scenario.stepTimeout ?? argValue(args, "step-timeout")) || BROWSER_STEP_DEFAULT_TIMEOUT_MS, environment, browserArgs } } function browserScenarioCaptures(scenario: BrowserScenarioInput, args: string[]): string[] { @@ -997,17 +1115,16 @@ function browserScenarioCaptures(scenario: BrowserScenarioInput, args: string[]) return captures.length > 0 ? captures : ["steps", "console", "errors", "html", "network", "screenshot", "dom-snapshot"] } -function browserScenarioRunPlan(scenario: BrowserScenarioInput, args: string[], url: string): BrowserRunPlan { +async function browserScenarioRunPlan(scenario: BrowserScenarioInput, args: string[], url: string, artifactRoot: string, requestedEnvironment: BrowserEnvironment): Promise { const captures = browserScenarioCaptures(scenario, args) - const steps = browserScenarioSteps(scenario, args) + const steps = await browserScenarioSteps(scenario, args) const assertions = browserScenarioAssertions(scenario) const actionSteps = [...steps, ...assertions] - const requestedViewport = browserScenarioViewport(scenario, args) - const device = scenario.device ?? (scenario.profile && scenario.profile !== "desktop-chrome" ? scenario.profile : undefined) ?? argValue(args, "device") - const locale = scenario.locale ?? argValue(args, "locale") + const requestedViewport = requestedEnvironment.viewport const prePageScript = scenario.prePageScript ?? browserScenarioObserverScript(scenario.observers) ?? argValue(args, "pre-page-script") const authRequest = browserScenarioAuthRequest(scenario.auth ?? argValue(args, "auth"), scenario.authUserId ?? argValue(args, "auth-user-id")) const shouldRunProbe = actionSteps.length === 0 || Boolean(prePageScript) || captures.some((capture) => capture === "performance" || capture === "memory") + const shouldRunActions = actionSteps.length > 0 || Object.keys(requestedEnvironment).length > 0 const plan: BrowserRunPlan = { profile: scenario.profile ?? "desktop-chrome", @@ -1020,11 +1137,15 @@ function browserScenarioRunPlan(scenario: BrowserScenarioInput, args: string[], capture: new Set(browserScenarioProbeCaptures(captures, actionSteps.length > 0)), waitFor: scenario.waitFor ?? argValue(args, "wait-for") ?? "domcontentloaded", durationMs: durationStringMs(scenario.duration ?? argValue(args, "duration")), - requestedViewport: requestedViewport ? parseBrowserViewport(requestedViewport, "viewport") : undefined, + requestedViewport, requestedContext: { - ...(device ? { device } : {}), - ...(locale ? { locale } : {}), - ...(requestedViewport ? { viewport: parseBrowserViewport(requestedViewport, "viewport") } : {}), + ...(requestedEnvironment.device ? { device: requestedEnvironment.device } : {}), + ...(requestedEnvironment.locale ? { locale: requestedEnvironment.locale } : {}), + ...(requestedEnvironment.timezone ? { timezone: requestedEnvironment.timezone } : {}), + ...(requestedEnvironment.geolocation ? { geolocation: requestedEnvironment.geolocation } : {}), + ...(requestedEnvironment.viewport ? { viewport: requestedEnvironment.viewport } : {}), + ...(requestedEnvironment.userAgent ? { userAgent: requestedEnvironment.userAgent } : {}), + ...(requestedEnvironment.permissions ? { permissions: requestedEnvironment.permissions } : {}), }, prePageScript, authRequest, @@ -1034,10 +1155,11 @@ function browserScenarioRunPlan(scenario: BrowserScenarioInput, args: string[], wallTimeoutMs: durationStringMs(scenario.timeout ?? argValue(args, "timeout")) || browserCommandLivenessPolicy().wallTimeoutMs, lifecycleSelectors: [], assertions: [], + storageStateImport: await browserStorageStateImportFromArgs(args, "wordpress.browser-scenario", artifactRoot), } } - if (actionSteps.length > 0) { + if (shouldRunActions) { const validation = validateBrowserInteractionScript(actionSteps) if (!validation.valid) { throw new Error(`wordpress.browser-scenario steps/assertions are invalid: ${validation.issues.map((issue) => `[${issue.index}] ${issue.message}`).join("; ")}`) @@ -1049,8 +1171,10 @@ function browserScenarioRunPlan(scenario: BrowserScenarioInput, args: string[], stepTimeoutMs: durationStringMs(scenario.stepTimeout ?? argValue(args, "step-timeout")) || BROWSER_STEP_DEFAULT_TIMEOUT_MS, totalTimeoutMs: durationStringMs(scenario.timeout ?? argValue(args, "timeout")) || BROWSER_SCRIPT_DEFAULT_TIMEOUT_MS, networkSettleTimeoutMs: durationArg(args, "network-settle-timeout", browserCommandLivenessPolicy().networkSettleTimeoutMs), - requestedViewport: requestedViewport ? parseBrowserViewport(requestedViewport, "viewport") : undefined, + requestedViewport, + requestedEnvironment, authRequest, + storageStateImport: await browserStorageStateImportFromArgs(args, "wordpress.browser-scenario", artifactRoot), maxDomSnapshotElements: positiveIntegerArg(args, "max-dom-snapshot-elements", 160), } } @@ -1058,6 +1182,26 @@ function browserScenarioRunPlan(scenario: BrowserScenarioInput, args: string[], return plan } +async function browserScenarioEnvironment(scenario: BrowserScenarioInput, args: string[]): Promise { + const argumentEnvironment = await browserEnvironmentFromArgs(args) + let profileEnvironment: BrowserEnvironment = {} + if (scenario.profile) { + const profile = BROWSER_PROBE_PROFILES[scenario.profile as keyof typeof BROWSER_PROBE_PROFILES] + if (!profile) throw new Error(`wordpress.browser-scenario unknown profile: ${scenario.profile}. Supported profiles: ${Object.keys(BROWSER_PROBE_PROFILES).join(", ")}`) + const profileViewport = profile.args.find((arg) => arg.startsWith("viewport="))?.slice("viewport=".length) + const profileDevice = profile.args.find((arg) => arg.startsWith("device="))?.slice("device=".length) + profileEnvironment = browserEnvironment({ ...(profileViewport ? { viewport: parseBrowserViewport(profileViewport, "viewport") } : {}), ...(profileDevice ? { device: profileDevice } : {}) }) + } + return browserEnvironment({ + ...argumentEnvironment, + ...profileEnvironment, + ...(scenario.environment ?? {}), + ...(scenario.device ? { device: scenario.device } : {}), + ...(scenario.locale ? { locale: scenario.locale } : {}), + ...(scenario.viewport ? { viewport: parseBrowserViewport(scenario.viewport, "viewport") } : {}), + }) +} + function browserScenarioProbeCaptures(captures: string[], actionsWillRun: boolean): string[] { const supported = new Set(["console", "errors", "html", "network", "performance", "memory", "screenshot"]) const selected = captures.filter((capture) => supported.has(capture) && (!actionsWillRun || capture === "performance" || capture === "memory")) @@ -1070,8 +1214,8 @@ function browserScenarioActionCaptures(captures: string[]): string[] { return selected.length > 0 ? selected : ["steps", "console", "errors", "html", "network", "screenshot", "dom-snapshot"] } -function browserScenarioSteps(scenario: BrowserScenarioInput, args: string[]): Array> { - const raw = scenario.steps ?? parseInlineJsonArrayArg(args, "steps-json") +async function browserScenarioSteps(scenario: BrowserScenarioInput, args: string[]): Promise>> { + const raw = scenario.steps ?? await parseJsonArrayArg(args, "steps-json") return (raw ?? []).map((step) => normalizeBrowserScenarioStep(step)) } @@ -1109,23 +1253,17 @@ function normalizeBrowserScenarioAssertion(assertion: Record): return assertion } -function parseInlineJsonArrayArg(args: string[], name: string): Array> | undefined { +async function parseJsonArrayArg(args: string[], name: string): Promise> | undefined> { const raw = argValue(args, name) - if (!raw || raw.startsWith("@")) { - return undefined - } - const parsed = JSON.parse(raw) as unknown + if (!raw) return undefined + const text = raw.startsWith("@") ? await readFile(resolveCommandPath(raw.slice(1)), "utf8") : raw + const parsed = JSON.parse(text) as unknown if (!Array.isArray(parsed)) { throw new Error(`wordpress.browser-scenario ${name} must be a JSON array`) } return parsed as Array> } -function browserScenarioViewport(scenario: BrowserScenarioInput, args: string[]): string | undefined { - if (scenario.viewport) return scenario.viewport - return argValue(args, "viewport") -} - function parseBrowserViewport(raw: string, name: string): { width: number; height: number } { const match = raw.trim().match(/^(\d+)x(\d+)$/i) if (!match) { diff --git a/packages/runtime-playground/src/browser-adaptive-explorer.ts b/packages/runtime-playground/src/browser-adaptive-explorer.ts index b3dc5b545..697ad2f42 100644 --- a/packages/runtime-playground/src/browser-adaptive-explorer.ts +++ b/packages/runtime-playground/src/browser-adaptive-explorer.ts @@ -161,7 +161,7 @@ export async function exploreAdaptiveBrowserStateMachine({ const accessibilityScan = accessibilityCollector && contract.accessibility?.cadence.includes("novel-state") && (newState || action.family === "keyboard") && !actionError ? await accessibilityCollector.scan({ phase: "novel-state", stateDigest: stabilized.state.digest, transitionId, action }) : undefined - const accessibilityFingerprints = accessibilityScan?.findings.map((finding) => finding.fingerprint) ?? [] + const accessibilityFingerprints = accessibilityScan?.findings.map((finding) => adaptiveOracleFingerprint(finding.fingerprint, contract)) ?? [] if (accessibilityScan && accessibilityScan.findings.length > 0 && onAccessibilityFindingEvidence) accessibilityScan.artifacts = await onAccessibilityFindingEvidence(accessibilityScan) fingerprints.push(...accessibilityFingerprints) fingerprints.sort() @@ -229,6 +229,8 @@ export async function exploreAdaptiveBrowserStateMachine({ expectedStateDigest: destination.digest, actions: path, resetPolicy: contract.resetPolicy, + environment: contract.environment, + environmentDigest: contract.environmentDigest, }, } if (artifactBytes(states, transitions, diagnostics, [finding]) + (accessibilityCollector ? Buffer.byteLength(stableJson(accessibilityCollector.evidence())) : 0) > adaptiveJsonArtifactBudget(contract, accessibilityCollector)) { @@ -276,7 +278,7 @@ export async function exploreAdaptiveBrowserStateMachine({ ...(accessibilityCollector ? { accessibility: accessibilityCollector.evidence() } : {}), diagnostics: diagnostics.slice(0, contract.descriptorLimits.maxDiagnostics), summary: { actions, states: states.size, transitions: transitions.length, revisits, errors, findings: findings.length, ...(exhausted ? { budgetExhausted: exhausted } : {}) }, - replay: { schema: BROWSER_ADAPTIVE_EXPLORATION_SCHEMA, seed: contract.seed, startUrl: contract.startUrl, contract }, + replay: { schema: BROWSER_ADAPTIVE_EXPLORATION_SCHEMA, seed: contract.seed, startUrl: contract.startUrl, environment: contract.environment, environmentDigest: contract.environmentDigest, contract }, } } @@ -359,7 +361,7 @@ async function captureAdaptiveState(page: Page, contract: BrowserAdaptiveExplora } if (descriptors.length > boundedDescriptors.length) diagnostics.push({ code: "browser_adaptive_descriptors_truncated", message: "Actionable descriptors were truncated at the per-state bound.", metadata: { discovered: descriptors.length, retained: boundedDescriptors.length } }) const descriptorDigest = browserAdaptiveDigest("descriptors", boundedDescriptors.map(stableDescriptor)) - const digest = browserAdaptiveDigest("state", { frames: semanticFrames.map(({ historyLength: _historyLength, ...frame }) => frame), descriptorDigest }) + const digest = browserAdaptiveDigest("state", { environmentDigest: contract.environmentDigest, frames: semanticFrames.map(({ historyLength: _historyLength, ...frame }) => frame), descriptorDigest }) const mainSemantic = semanticFrames.find((frame) => frame.id === "document") return { state: { @@ -527,7 +529,7 @@ async function minimizeAdaptiveFinding(page: Page, baseUrl: string, contract: Br const replay = await restoreAdaptivePath(page, baseUrl, contract, candidate, signal, navigationScope, accessibilityCollector) executed += replay.executed keyboardExecuted += replay.keyboardExecuted - const fingerprints = adaptiveOracleEvidence(consoleErrorRecords(observations.consoleMessages.slice(beforeConsole)), observations.network.slice(beforeNetwork), errorMessages(observations.errors.slice(beforeErrors)), contract, networkPolicy).fingerprints.concat(replay.finalAccessibilityFingerprints) + const fingerprints = adaptiveOracleEvidence(consoleErrorRecords(observations.consoleMessages.slice(beforeConsole)), observations.network.slice(beforeNetwork), errorMessages(observations.errors.slice(beforeErrors)), contract, networkPolicy).fingerprints.concat(replay.finalAccessibilityFingerprints.map((fingerprint) => adaptiveOracleFingerprint(fingerprint, contract))) if (replay.state && fingerprints.includes(finding.fingerprint) && (!finding.stateDigest || replay.state.digest === finding.stateDigest)) { current = candidate reduced = true @@ -603,7 +605,7 @@ function adaptiveOracleEvidence(consoleRecords: Record[], netwo const messages = [...consoleMessages, ...pageErrors, ...networkMessages] const networkFailureSummary = classifiedFailures.length > 0 ? boundedNetworkFailureEvidence(classifiedFailures, contract, retention) : undefined return { - fingerprints: [...new Set(messages.map((message) => browserAdaptiveDigest("oracle", message)))].sort(), + fingerprints: [...new Set(messages.map((message) => adaptiveOracleFingerprint(message, contract)))].sort(), networkFailures: networkFailureSummary?.failures ?? [], networkFailureSummary: networkFailureSummary?.summary, errorCount: consoleMessages.length + pageErrors.length + networkMessages.length, @@ -683,15 +685,19 @@ function accessibilityRequirementsUnavailable(evidence: ReturnType ({ - fingerprint: item.fingerprint, + fingerprint: adaptiveOracleFingerprint(item.fingerprint, contract), stateDigest: item.stateDigest, transitionId, originalPath: path, minimizedPath: path, - replay: { schema: BROWSER_ADAPTIVE_EXPLORATION_SCHEMA, seed: contract.seed, startUrl: contract.startUrl, expectedFingerprint: item.fingerprint, expectedStateDigest: item.stateDigest, actions: path, resetPolicy: contract.resetPolicy }, + replay: { schema: BROWSER_ADAPTIVE_EXPLORATION_SCHEMA, seed: contract.seed, startUrl: contract.startUrl, expectedFingerprint: adaptiveOracleFingerprint(item.fingerprint, contract), expectedStateDigest: item.stateDigest, actions: path, resetPolicy: contract.resetPolicy, environment: contract.environment, environmentDigest: contract.environmentDigest }, })) } +function adaptiveOracleFingerprint(value: string, contract: BrowserAdaptiveExplorationContract): string { + return browserAdaptiveDigest("oracle", { environmentDigest: contract.environmentDigest, value }) +} + function appendDiagnostics(target: BrowserAdaptiveExplorationResult["diagnostics"], incoming: BrowserAdaptiveExplorationResult["diagnostics"], maximum: number): void { const remaining = Math.max(0, maximum - target.length) if (remaining > 0) target.push(...incoming.slice(0, remaining)) diff --git a/packages/runtime-playground/src/browser-artifacts.ts b/packages/runtime-playground/src/browser-artifacts.ts index 329f9edb2..24b5a3d64 100644 --- a/packages/runtime-playground/src/browser-artifacts.ts +++ b/packages/runtime-playground/src/browser-artifacts.ts @@ -1,5 +1,5 @@ import { join } from "node:path" -import { artifactManifestFile, type ArtifactManifestFile, type ArtifactManifestFileOptions, type ArtifactReviewBrowserSummary, type BrowserGeolocation } from "@automattic/wp-codebox-core" +import { artifactManifestFile, type ArtifactManifestFile, type ArtifactManifestFileOptions, type ArtifactReviewBrowserSummary, type BrowserEnvironment, type BrowserEnvironmentCapabilityResult, type BrowserGeolocation, type ResolvedBrowserEnvironment } from "@automattic/wp-codebox-core" import type { PlaygroundPreviewProxyDiagnostics } from "./preview-server.js" import type { Request } from "playwright" @@ -207,6 +207,26 @@ export interface BrowserArtifactSummary { replay: string } capabilities?: BrowserProbeCapabilityDiagnostics + environment?: { + requested: BrowserEnvironment + resolved: BrowserEnvironment + observed?: { + viewport: { width: number; height: number } + deviceScaleFactor: number + hasTouch: boolean + maxTouchPoints: number + userAgent: string + locale: string + timezone?: string + online: boolean + geolocationPermission?: "granted" | "denied" | "prompt" | "unsupported" + geolocation?: { latitude: number; longitude: number; accuracy: number } + } + provider?: ResolvedBrowserEnvironment["provider"] + capabilities: BrowserEnvironmentCapabilityResult[] + unsupported: string[] + inconclusive: string[] + } replayability: BrowserProbeReplayability screenshot: boolean visualCompare?: { diff --git a/packages/runtime-playground/src/browser-environment-matrix.ts b/packages/runtime-playground/src/browser-environment-matrix.ts index 64366d97e..176cb08cc 100644 --- a/packages/runtime-playground/src/browser-environment-matrix.ts +++ b/packages/runtime-playground/src/browser-environment-matrix.ts @@ -12,10 +12,13 @@ import { type ResolvedBrowserEnvironment, } from "@automattic/wp-codebox-core" import type { Browser, BrowserContext, BrowserContextOptions, Page } from "playwright" +import type { BrowserArtifactSummary } from "./browser-artifacts.js" export const PLAYWRIGHT_BROWSER_ENVIRONMENT_CAPABILITIES = [ "browser.environment.viewport", "browser.environment.device", + "browser.environment.user-agent", + "browser.environment.permissions", "browser.environment.device-scale-factor", "browser.environment.mobile", "browser.environment.touch", @@ -49,6 +52,7 @@ export interface PlaywrightBrowserEnvironmentOptions { channel?: string networkProfiles?: Record cpuProfiles?: Record + contextOptions?: BrowserContextOptions } export interface PlaywrightBrowserEnvironmentExecutionInput { @@ -61,6 +65,19 @@ export interface PlaywrightBrowserEnvironmentExecutionInput { signal: AbortSignal } +export interface PlaywrightBrowserEnvironmentRuntime { + context: BrowserContext + page: Page + close(): Promise +} + +export interface PlaywrightBrowserEnvironmentSession { + browser: Browser + requested: BrowserEnvironment + resolved: ResolvedBrowserEnvironment + runtime: PlaywrightBrowserEnvironmentRuntime +} + export async function resolvePlaywrightBrowserEnvironment(cell: BrowserEnvironmentCell, browser: Browser, options: PlaywrightBrowserEnvironmentOptions = {}): Promise { const { devices } = await import("playwright") const requested = cell.requested @@ -72,6 +89,8 @@ export async function resolvePlaywrightBrowserEnvironment(cell: BrowserEnvironme if (requested.viewport) exact("browser.environment.viewport") if (requested.device) device ? exact("browser.environment.device") : unsupported("browser.environment.device", `Unknown device profile: ${requested.device}`) + if (requested.userAgent) exact("browser.environment.user-agent") + if (requested.permissions) exact("browser.environment.permissions") if (requested.deviceScaleFactor !== undefined) exact("browser.environment.device-scale-factor") if (requested.isMobile !== undefined) exact("browser.environment.mobile") if (requested.hasTouch !== undefined) exact("browser.environment.touch") @@ -94,7 +113,11 @@ export async function resolvePlaywrightBrowserEnvironment(cell: BrowserEnvironme if (requested.networkProfile) options.networkProfiles?.[requested.networkProfile] ? emulated("browser.environment.network-profile", "Latency and throughput are applied through the browser debugging protocol.") : unsupported("browser.environment.network-profile", `Unknown network profile: ${requested.networkProfile}`) if (requested.cpuProfile) options.cpuProfiles?.[requested.cpuProfile] ? emulated("browser.environment.cpu-profile", "CPU slowdown is applied through the browser debugging protocol.") : unsupported("browser.environment.cpu-profile", `Unknown CPU profile: ${requested.cpuProfile}`) - const effective = mergeDeviceEnvironment(requested, device) + const merged = mergeDeviceEnvironment(requested, device) + const effectivePermissions = requested.permissions || requested.geolocation + ? [...new Set([...(requested.permissions ?? []).filter((permission) => permission !== "geolocation"), ...(requested.geolocation?.permission === "granted" ? ["geolocation"] : [])])] + : undefined + const effective = { ...merged, ...(effectivePermissions ? { permissions: effectivePermissions } : {}) } return { effective, capabilities: capabilities.sort((left, right) => left.id.localeCompare(right.id)), @@ -102,17 +125,23 @@ export async function resolvePlaywrightBrowserEnvironment(cell: BrowserEnvironme } } -export async function createPlaywrightBrowserEnvironmentContext(browser: Browser, resolved: ResolvedBrowserEnvironment, options: PlaywrightBrowserEnvironmentOptions = {}): Promise<{ context: BrowserContext; page: Page; close(): Promise }> { +export function browserEnvironmentCell(requested: BrowserEnvironment): BrowserEnvironmentCell { + return { id: "browser-environment", index: 0, seed: "browser-environment", selections: {}, requested, requiredCapabilities: [], optionalCapabilities: [] } +} + +export async function createPlaywrightBrowserEnvironmentContext(browser: Browser, resolved: ResolvedBrowserEnvironment, options: PlaywrightBrowserEnvironmentOptions = {}): Promise { const { devices } = await import("playwright") const environment = resolved.effective const device = environment.device ? devices[environment.device] : undefined const viewport = orientedViewport(environment.viewport ?? device?.viewport ?? undefined, environment.orientation) const contextOptions: BrowserContextOptions = { ...(device ?? {}), + ...(options.contextOptions ?? {}), ...(viewport ? { viewport, screen: viewport } : {}), ...(environment.deviceScaleFactor !== undefined ? { deviceScaleFactor: environment.deviceScaleFactor } : {}), ...(environment.isMobile !== undefined ? { isMobile: environment.isMobile } : {}), ...(environment.hasTouch !== undefined ? { hasTouch: environment.hasTouch } : {}), + ...(environment.userAgent ? { userAgent: environment.userAgent } : {}), ...(environment.locale ? { locale: environment.locale } : {}), ...(environment.timezone ? { timezoneId: environment.timezone } : {}), ...(environment.colorScheme ? { colorScheme: environment.colorScheme } : {}), @@ -121,7 +150,7 @@ export async function createPlaywrightBrowserEnvironmentContext(browser: Browser ...(environment.contrast ? { contrast: environment.contrast } : {}), ...(environment.online !== undefined ? { offline: !environment.online } : {}), ...(environment.geolocation ? { geolocation: { latitude: environment.geolocation.latitude, longitude: environment.geolocation.longitude, ...(environment.geolocation.accuracy !== undefined ? { accuracy: environment.geolocation.accuracy } : {}) } } : {}), - ...(environment.geolocation?.permission === "granted" ? { permissions: ["geolocation"] } : {}), + ...(environment.permissions?.length || environment.geolocation?.permission === "granted" ? { permissions: [...new Set([...(environment.permissions ?? []).filter((permission) => permission !== "geolocation"), ...(environment.geolocation?.permission === "granted" ? ["geolocation"] : [])])] } : {}), } const context = await browser.newContext(contextOptions) const page = await context.newPage() @@ -130,6 +159,48 @@ export async function createPlaywrightBrowserEnvironmentContext(browser: Browser return { context, page, close: async () => { await geolocationPermissionCleanup?.(); await context.close() } } } +export async function observePlaywrightBrowserEnvironment(page: Page, requested: BrowserEnvironment, resolved: ResolvedBrowserEnvironment): Promise> { + const observed: NonNullable["observed"]> | undefined = await page.evaluate(async () => { + let geolocationPermission: "granted" | "denied" | "prompt" | "unsupported" | undefined + try { + const state = (await navigator.permissions.query({ name: "geolocation" })).state + geolocationPermission = state === "granted" || state === "denied" || state === "prompt" ? state : "unsupported" + } catch { + geolocationPermission = "unsupported" + } + return { + viewport: { width: innerWidth, height: innerHeight }, + deviceScaleFactor: devicePixelRatio, + hasTouch: navigator.maxTouchPoints > 0, + maxTouchPoints: navigator.maxTouchPoints, + userAgent: navigator.userAgent, + locale: navigator.language, + timezone: Intl.DateTimeFormat().resolvedOptions().timeZone || undefined, + online: navigator.onLine, + ...(geolocationPermission ? { geolocationPermission } : {}), + } + }).catch(() => undefined) + const observedGeolocation = requested.geolocation && observed?.geolocationPermission === "granted" + ? await page.evaluate(() => new Promise<{ latitude: number; longitude: number; accuracy: number } | undefined>((resolve) => navigator.geolocation.getCurrentPosition( + ({ coords }) => resolve({ latitude: coords.latitude, longitude: coords.longitude, accuracy: coords.accuracy }), + () => resolve(undefined), + { timeout: 1_000 }, + ))).catch(() => undefined) + : undefined + if (observed && observedGeolocation) observed.geolocation = observedGeolocation + const unsupported = resolved.capabilities.filter(({ fidelity }) => fidelity === "unsupported").map(({ id }) => id) + const inconclusive: string[] = [] + if (requested.device) inconclusive.push("browser.environment.device") + if (requested.isMobile !== undefined) inconclusive.push("browser.environment.mobile") + if (requested.geolocation && !observed?.geolocation) inconclusive.push("browser.environment.geolocation.coordinates") + if (requested.permissions?.some((permission) => permission !== "geolocation")) inconclusive.push("browser.environment.permissions") + for (const [key, capability] of [["orientation", "browser.environment.orientation"], ["zoom", "browser.environment.zoom"], ["colorScheme", "browser.environment.color-scheme"], ["reducedMotion", "browser.environment.reduced-motion"], ["forcedColors", "browser.environment.forced-colors"], ["contrast", "browser.environment.contrast"], ["networkProfile", "browser.environment.network-profile"], ["cpuProfile", "browser.environment.cpu-profile"], ["clock", "browser.environment.clock"], ["capabilities", "browser.environment.capability-state"]] as const) { + if (requested[key] !== undefined) inconclusive.push(capability) + } + if (!observed) inconclusive.push("browser.environment.observation") + return { requested, resolved: resolved.effective, ...(observed ? { observed } : {}), provider: resolved.provider, capabilities: resolved.capabilities, unsupported, inconclusive } +} + export async function applyPlaywrightGeolocationPermission(page: Page, state: "denied"): Promise<() => Promise> { if (page.context().browser()?.browserType().name() !== "chromium") throw new Error("Explicit denied geolocation permission is unsupported by this browser provider.") const session = await page.context().newCDPSession(page) diff --git a/packages/runtime-playground/src/browser-multi-actor-scenario-runner.ts b/packages/runtime-playground/src/browser-multi-actor-scenario-runner.ts index 066772f80..c37415fbd 100644 --- a/packages/runtime-playground/src/browser-multi-actor-scenario-runner.ts +++ b/packages/runtime-playground/src/browser-multi-actor-scenario-runner.ts @@ -1,5 +1,5 @@ import type { Page } from "playwright" -import { type BrowserMultiActorScenario, type RuntimeCreateSpec } from "@automattic/wp-codebox-core" +import { type BrowserEnvironment, type BrowserMultiActorScenario, type RuntimeCreateSpec } from "@automattic/wp-codebox-core" import { now } from "@automattic/wp-codebox-core/internals" import { BrowserArtifactSession } from "./browser-artifact-session.js" import { BrowserCommandArtifactError } from "./browser-command-artifact-error.js" @@ -14,12 +14,13 @@ import type { PlaygroundCliServer } from "./preview-server.js" import { BrowserMultiActorScenarioError, runBrowserMultiActorScenario, type BrowserMultiActorClient, type BrowserMultiActorScenarioResult } from "./browser-multi-actor-scenario.js" import { wordpressFixtureUserPhpCode, wordpressUserSessionFromCommandArgs, type WordPressFixtureUserSpec } from "./wordpress-user-sessions.js" import type { BrowserArtifact, BrowserProbeErrorRecord, BrowserProbeNetworkRecord } from "./browser-artifacts.js" +import { browserEnvironmentCell, createPlaywrightBrowserEnvironmentContext, observePlaywrightBrowserEnvironment, resolvePlaywrightBrowserEnvironment } from "./browser-environment-matrix.js" const DEFAULT_STEP_TIMEOUT_MS = 15_000 export async function runBrowserMultiActorScenarioCommand(input: { artifactRoot: string - scenario: BrowserMultiActorScenario & { url: string; captures?: string[]; stepTimeoutMs?: number } + scenario: BrowserMultiActorScenario & { url: string; captures?: string[]; stepTimeoutMs?: number; environment: BrowserEnvironment; browserArgs: string[] } runtimeSpec: RuntimeCreateSpec runPlaygroundCommand?: (command: string, server: PlaygroundCliServer, options: { code: string } | { scriptPath: string }) => Promise server: PlaygroundCliServer @@ -29,17 +30,22 @@ export async function runBrowserMultiActorScenarioCommand(input: { const captures = new Set([...(scenario.captures ?? ["steps", "console", "errors", "network", "screenshot"]), "trace"]) const artifacts = new BrowserArtifactSession(artifactRoot, "files/browser", { source: "wordpress.browser-scenario", operation: "browser-multi-actor-scenario" }) const routeHost = runtimeSpec.preview?.siteUrl ? new URL(runtimeSpec.preview.siteUrl).hostname : "" - const topology = browserPreviewTopology(routeHost ? [`route-host=${routeHost}`] : [], runtimeSpec, server.serverUrl, server.previewProxyDiagnostics?.targetOrigin) + const browserArgs = routeHost && !scenario.browserArgs.some((arg) => arg.startsWith("route-host=")) ? [...scenario.browserArgs, `route-host=${routeHost}`] : scenario.browserArgs + const topology = browserPreviewTopology(browserArgs, runtimeSpec, server.serverUrl, server.previewProxyDiagnostics?.targetOrigin) const browser = await launchChromiumBrowser() const evidence: Record = {} let result: BrowserMultiActorScenarioResult | undefined let failure: Error | undefined + let resolvedEnvironment: Awaited> try { const previewReadinessError = browserPreviewReadinessError(topology.preview) if (previewReadinessError) { throw previewReadinessError } + resolvedEnvironment = await resolvePlaywrightBrowserEnvironment(browserEnvironmentCell(scenario.environment), browser) + const unsupportedEnvironment = resolvedEnvironment.capabilities.filter(({ fidelity }) => fidelity === "unsupported").map(({ id }) => id) + if (unsupportedEnvironment.length > 0) throw new Error(`Multi-actor wordpress.browser-scenario browser environment is unsupported: ${unsupportedEnvironment.join(", ")}`) const clientEntries: Array<[string, BrowserMultiActorClient]> = [] const actorPages: Array<{ actor: string; page: Pick }> = [] // Playground PHP commands share one runtime endpoint, so provision identities @@ -48,15 +54,16 @@ export async function runBrowserMultiActorScenarioCommand(input: { const session = wordpressUserSessionFromCommandArgs([`session=${actor.userSession}`], runtimeSpec) if (!session) throw new Error(`Actor ${actor.name} requires user session ${actor.userSession}`) const userId = await actorUserId(actor.name, session.user.userId, session.user, runtimeSpec, runPlaygroundCommand, server) - const context = await browser.newContext(topology.contextOptions()) + const environmentRuntime = await createPlaywrightBrowserEnvironmentContext(browser, resolvedEnvironment, { contextOptions: topology.contextOptions() }) + const context = environmentRuntime.context await routeBrowserPreviewContextNetwork(context, topology.networkPolicy, topology.origins.localProxyOrigin) - const page = await context.newPage() + const page = environmentRuntime.page await context.tracing.start({ screenshots: true, snapshots: true }) await installWordPressAdminAuthCookies({ command: "wordpress.browser-scenario", cookieUrls: topology.authCookieUrls([topology.resolveUrl(scenario.url)]), page, runPlaygroundCommand, runtimeSpec, server, userId }) - const actorEvidence = evidence[actor.name] = { console: [], errors: [], network: [], steps: [], files: {} } + const actorEvidence = evidence[actor.name] = { console: [], errors: [], network: [], steps: [], files: {}, environment: await observePlaywrightBrowserEnvironment(page, scenario.environment, resolvedEnvironment) } const networkTasks: Array> = [] attachBrowserCaptureListeners({ captureConsole: captures.has("console"), captureErrors: captures.has("errors"), captureNetwork: true, consoleMessages: actorEvidence.console, errors: actorEvidence.errors, network: actorEvidence.network, networkTasks, page }) - clientEntries.push([actor.name, actorClient({ actor: actor.name, artifacts, captures, context, evidence: actorEvidence, networkTasks, page, scenario, previewOrigin: topology.preview.effectiveOrigin })]) + clientEntries.push([actor.name, actorClient({ actor: actor.name, artifacts, captures, context, closeEnvironment: environmentRuntime.close, evidence: actorEvidence, networkTasks, page, scenario, previewOrigin: topology.preview.effectiveOrigin })]) actorPages.push({ actor: actor.name, page }) } const clients = Object.fromEntries(clientEntries) @@ -82,7 +89,7 @@ export async function runBrowserMultiActorScenarioCommand(input: { await artifacts.writeJson("waterfall", "multi-actor-waterfall.json", network) const target = topology.resolveUrl(scenario.url) const traces = Object.values(evidence).map((actor) => actor.files.trace).filter((path): path is string => Boolean(path)) - const artifact = { artifactType: "scenario" as const, requestedUrl: target, url: target, preview: topology.preview, ...topology.origins, files: { summary: "files/browser/multi-actor-scenario-summary.json", steps: "files/browser/multi-actor-events.json", network: "files/browser/multi-actor-network.json", requestCoverage: "files/browser/multi-actor-request-coverage.json", waterfall: "files/browser/multi-actor-waterfall.json", ...(traces.length > 0 ? { traces } : {}) }, summary: { actions: scenario.actions.length, steps: scenario.actions.length, consoleMessages: Object.values(evidence).reduce((total, actor) => total + actor.console.length, 0), errors: Object.values(evidence).reduce((total, actor) => total + actor.errors.length, 0), finalUrl: target, htmlSnapshot: false, networkEvents: network.length, replayability: browserProbeReplayability(captures), screenshot: captures.has("screenshot"), viewport: null, multiActor: { seed: scenario.seed, finalState: result?.finalState ?? "failed", actors: Object.keys(evidence), replay: "files/browser/multi-actor-replay.json" } } } satisfies BrowserArtifact + const artifact = { artifactType: "scenario" as const, requestedUrl: target, url: target, preview: topology.preview, ...topology.origins, files: { summary: "files/browser/multi-actor-scenario-summary.json", steps: "files/browser/multi-actor-events.json", network: "files/browser/multi-actor-network.json", requestCoverage: "files/browser/multi-actor-request-coverage.json", waterfall: "files/browser/multi-actor-waterfall.json", ...(traces.length > 0 ? { traces } : {}) }, summary: { actions: scenario.actions.length, steps: scenario.actions.length, consoleMessages: Object.values(evidence).reduce((total, actor) => total + actor.console.length, 0), errors: Object.values(evidence).reduce((total, actor) => total + actor.errors.length, 0), finalUrl: target, htmlSnapshot: false, networkEvents: network.length, replayability: browserProbeReplayability(captures), screenshot: captures.has("screenshot"), viewport: null, environment: Object.values(evidence)[0]?.environment, multiActor: { seed: scenario.seed, finalState: result?.finalState ?? "failed", actors: Object.keys(evidence), replay: "files/browser/multi-actor-replay.json" } } } satisfies BrowserArtifact if (failure) throw new BrowserCommandArtifactError(`wordpress.browser-scenario failed: ${failure.message}`, artifact) return { artifact, output: `${JSON.stringify({ command: "wordpress.browser-scenario", files: artifact.files, summary: artifact.summary, scenario: summary }, null, 2)}\n` } } @@ -107,19 +114,21 @@ interface ActorEvidence { network: BrowserProbeNetworkRecord[] steps: Array> files: Record + environment: NonNullable } -function actorClient(input: { actor: string; artifacts: BrowserArtifactSession; captures: Set; context: import("playwright").BrowserContext; evidence: ActorEvidence; networkTasks: Array>; page: import("playwright").Page; scenario: BrowserMultiActorScenario & { stepTimeoutMs?: number }; previewOrigin: string }): BrowserMultiActorClient { - const { actor, artifacts, captures, context, evidence, networkTasks, page, scenario, previewOrigin } = input +function actorClient(input: { actor: string; artifacts: BrowserArtifactSession; captures: Set; context: import("playwright").BrowserContext; closeEnvironment: () => Promise; evidence: ActorEvidence; networkTasks: Array>; page: import("playwright").Page; scenario: BrowserMultiActorScenario & { stepTimeoutMs?: number }; previewOrigin: string }): BrowserMultiActorClient { + const { actor, artifacts, captures, context, closeEnvironment, evidence, networkTasks, page, scenario, previewOrigin } = input return { async execute(action) { const startedAt = now() try { - await executeBrowserInteractionStep(page, action.step, previewOrigin, scenario.stepTimeoutMs ?? DEFAULT_STEP_TIMEOUT_MS, async (name, write) => { + const outcome = await executeBrowserInteractionStep(page, action.step, previewOrigin, scenario.stepTimeoutMs ?? DEFAULT_STEP_TIMEOUT_MS, async (name, write) => { const fileName = `${actor}-${name}` await artifacts.writeGenerated("screenshot", fileName, write) return { path: artifacts.path(fileName), isDefault: false } }) + if (outcome.assertion && !outcome.assertion.passed) throw new Error(`Actor ${actor} ${action.step.kind} assertion failed for action ${action.id}`) evidence.steps.push({ action: action.id, status: "ok", startedAt, finalUrl: page.url() }) } catch (error) { evidence.steps.push({ action: action.id, status: "failed", startedAt, finalUrl: page.url(), error: error instanceof Error ? error.message : String(error) }) @@ -152,7 +161,7 @@ function actorClient(input: { actor: string; artifacts: BrowserArtifactSession; } } } finally { - await context.close() + await closeEnvironment() } }, } diff --git a/packages/runtime-playground/src/browser-probe-runner.ts b/packages/runtime-playground/src/browser-probe-runner.ts index 829e0e9e3..1b79afa37 100644 --- a/packages/runtime-playground/src/browser-probe-runner.ts +++ b/packages/runtime-playground/src/browser-probe-runner.ts @@ -1,4 +1,4 @@ -import { BROWSER_PROBE_BROWSER_VALUES, BROWSER_PROBE_CAPTURE_VALUES, BROWSER_PROBE_CHROMIUM_PROFILE_IDS, BROWSER_PROBE_PROFILES, BROWSER_PROBE_THROTTLE_PROFILE_IDS, browserGeolocation, redactError, type BrowserGeolocationPermissionState, type BrowserProbeProfileDefinition, type ExecutionSpec, type RuntimeCreateSpec } from "@automattic/wp-codebox-core" +import { BROWSER_PROBE_BROWSER_VALUES, BROWSER_PROBE_CAPTURE_VALUES, BROWSER_PROBE_CHROMIUM_PROFILE_IDS, BROWSER_PROBE_PROFILES, BROWSER_PROBE_THROTTLE_PROFILE_IDS, browserEnvironment, browserGeolocation, redactError, type BrowserGeolocationPermissionState, type BrowserProbeProfileDefinition, type ExecutionSpec, type RuntimeCreateSpec } from "@automattic/wp-codebox-core" import { BrowserArtifactSession } from "./browser-artifact-session.js" import { BrowserCommandArtifactError } from "./browser-command-artifact-error.js" import type { BrowserArtifactFiles, BrowserProbeArtifact, BrowserProbeAuthSummary, BrowserProbeCapabilityDiagnostics, BrowserProbeCheckpointRecord, BrowserProbeContextDetails, BrowserProbeErrorRecord, BrowserProbeLifecycleArtifact, BrowserProbeMemoryArtifact, BrowserProbeNetworkRecord, BrowserProbePerformanceArtifact, BrowserProbeScriptMetadata, BrowserProbeViewport, BrowserProbeWebSocketRecord, BrowserWordPressDiagnosticsSummary } from "./browser-artifacts.js" @@ -13,7 +13,7 @@ import type { PlaygroundRunResponse } from "./playground-command-errors.js" import type { PlaygroundCliServer } from "./preview-server.js" import { browserAuthRequest, browserProbeWaterfallArtifact, browserProbeWebSocketArtifact, browserRedirectDiagnosticsArtifact, browserRequestCoverageArtifact, browserStorageStateAuthSummary, browserStorageStateImportFromArgs, createBrowserProbeProgressTracker, fileSha256, installWordPressAdminAuthCookies, now, sha256, withBrowserProbeLiveness, normalizeBrowserProbeScriptCheckpoint, type BrowserCommandProgressEvent, type BrowserProbeScriptCheckpoint, type BrowserStorageStateImport } from "./browser-probe-support.js" import { BrowserProbeSessionResultBuilder, browserProbeCaptureSelection } from "./browser-probe-session-result-builder.js" -import { applyPlaywrightGeolocationPermission } from "./browser-environment-matrix.js" +import { browserEnvironmentCell, createPlaywrightBrowserEnvironmentContext, resolvePlaywrightBrowserEnvironment, type PlaywrightBrowserEnvironmentRuntime, type PlaywrightBrowserEnvironmentSession } from "./browser-environment-matrix.js" const BROWSER_PROBE_PROFILE_OVERRIDES = new Set(["browser", "device", "geolocation-accuracy", "geolocation-latitude", "geolocation-longitude", "geolocation-permission", "locale", "permissions", "throttle", "timezone", "user-agent", "viewport"]) @@ -98,6 +98,7 @@ export async function runBrowserProbeCommand({ spec, onProgress, diagnosticProviders, + session, }: { abortSignal?: AbortSignal artifactRoot: string @@ -109,9 +110,10 @@ export async function runBrowserProbeCommand({ spec: ExecutionSpec onProgress?: (event: BrowserCommandProgressEvent) => void diagnosticProviders?: BrowserProbeDiagnosticProvider[] + session?: PlaywrightBrowserEnvironmentSession }): Promise<{ artifact: BrowserProbeArtifact; artifacts?: BrowserProbeArtifact[]; output: string }> { if (plan) { - return runSingleBrowserProbeCommand({ abortSignal, artifactRoot, command, plan, runtimeSpec, runPlaygroundCommand, server, spec, browserFilesDirectory: "files/browser", onProgress, diagnosticProviders }) + return runSingleBrowserProbeCommand({ abortSignal, artifactRoot, command, plan, runtimeSpec, runPlaygroundCommand, server, spec, browserFilesDirectory: "files/browser", onProgress, diagnosticProviders, session }) } const profileIds = browserProbeProfileIds(spec.args ?? []) @@ -189,6 +191,7 @@ export async function runSingleBrowserProbeCommand({ profileId, onProgress, diagnosticProviders, + session, }: { abortSignal?: AbortSignal artifactRoot: string @@ -202,6 +205,7 @@ export async function runSingleBrowserProbeCommand({ profileId?: string onProgress?: (event: BrowserCommandProgressEvent) => void diagnosticProviders?: BrowserProbeDiagnosticProvider[] + session?: PlaywrightBrowserEnvironmentSession }): Promise<{ artifact: BrowserProbeArtifact; output: string }> { const args = spec.args ?? [] const runPlan = plan ?? await browserProbeRunPlanFromArgs(args, profileId, artifactRoot) @@ -266,7 +270,7 @@ export async function runSingleBrowserProbeCommand({ if (requestedContext.device && !deviceProfile) { throw new Error(`wordpress.browser-probe unknown Playwright device profile: ${requestedContext.device}`) } - const browser = await launchChromiumBrowser() + const browser = session?.browser ?? await launchChromiumBrowser() const browserMetadata = chromiumBrowserMetadata(browser) let finalUrl = targetUrl let windowLocationOrigin: string | undefined @@ -282,7 +286,7 @@ export async function runSingleBrowserProbeCommand({ let contextDetails: BrowserProbeContextDetails | undefined let authSummary: BrowserProbeAuthSummary | undefined let capabilityDiagnostics: BrowserProbeCapabilityDiagnostics | undefined - let geolocationPermissionCleanup: (() => Promise) | undefined + let environmentRuntime: PlaywrightBrowserEnvironmentRuntime | undefined let assertionResults: import("./browser-artifacts.js").BrowserStepAssertion[] = [] let pendingError: Error | undefined let artifact: BrowserProbeArtifact | undefined @@ -302,25 +306,28 @@ export async function runSingleBrowserProbeCommand({ throw pendingError } const contextPermissions = browserProbeContextPermissions(requestedContext) - context = browserPreviewNeedsContextRouting(networkPolicy) || !!storageStateImport || requestedContext.device || requestedContext.geolocation || requestedContext.locale || requestedContext.timezone || requestedContext.userAgent || contextPermissions.length > 0 - ? await browser.newContext({ - ...topology.contextOptions(), - ...(deviceProfile ?? {}), - ...(storageStateImport ? { storageState: storageStateImport.storageState } : {}), + if (session) { + environmentRuntime = session.runtime + } else { + const environment = browserEnvironment({ + ...(requestedContext.device ? { device: requestedContext.device } : {}), + ...(requestedViewport ? { viewport: requestedViewport } : {}), ...(requestedContext.locale ? { locale: requestedContext.locale } : {}), - ...(requestedContext.timezone ? { timezoneId: requestedContext.timezone } : {}), + ...(requestedContext.timezone ? { timezone: requestedContext.timezone } : {}), ...(requestedContext.userAgent ? { userAgent: requestedContext.userAgent } : {}), - ...(requestedContext.geolocation ? { geolocation: { latitude: requestedContext.geolocation.latitude, longitude: requestedContext.geolocation.longitude, ...(requestedContext.geolocation.accuracy !== undefined ? { accuracy: requestedContext.geolocation.accuracy } : {}) } } : {}), + ...(contextPermissions.length > 0 ? { permissions: contextPermissions } : {}), + ...(requestedContext.geolocation ? { geolocation: requestedContext.geolocation } : {}), }) - : null - if (context && contextPermissions.length > 0) { - await context.grantPermissions(contextPermissions) + const resolved = await resolvePlaywrightBrowserEnvironment(browserEnvironmentCell(environment), browser) + const unsupported = resolved.capabilities.filter(({ fidelity }) => fidelity === "unsupported").map(({ id }) => id) + if (unsupported.length > 0) throw new Error(`${command} browser environment is unsupported: ${unsupported.join(", ")}`) + environmentRuntime = await createPlaywrightBrowserEnvironmentContext(browser, resolved, { contextOptions: { ...topology.contextOptions(), ...(storageStateImport ? { storageState: storageStateImport.storageState } : {}) } }) } - if (context && browserPreviewNeedsContextRouting(networkPolicy)) { + context = environmentRuntime.context + if (context && browserPreviewNeedsContextRouting(networkPolicy) && !session) { await routeBrowserPreviewContextNetwork(context, networkPolicy, topology.origins.localProxyOrigin, routeTracker) } - page = context ? await context.newPage() : await browser.newPage() - if (requestedContext.geolocation?.permission === "denied") geolocationPermissionCleanup = await applyPlaywrightGeolocationPermission(page, "denied") + page = environmentRuntime.page if (onProgress) { await page.exposeFunction("__wpCodeboxProbeCheckpointEvent", (checkpoint: unknown) => { const normalized = normalizeBrowserProbeScriptCheckpoint(checkpoint) @@ -337,9 +344,6 @@ export async function runSingleBrowserProbeCommand({ if (authRequest) { authSummary = await installWordPressAdminAuthCookies({ command, cookieUrls: topology.authCookieUrls([targetUrl]), page, runPlaygroundCommand, runtimeSpec, server, userId: authRequest.userId }) } - if (requestedViewport) { - await page.setViewportSize(requestedViewport) - } if (throttleProfile) { await applyBrowserProbeThrottleProfile(page, throttleProfile) } @@ -503,8 +507,10 @@ export async function runSingleBrowserProbeCommand({ } } await settleBrowserNetworkTasks(networkTasks, livenessPolicy.networkSettleTimeoutMs) - await geolocationPermissionCleanup?.() - await browser.close() + if (!session) { + await environmentRuntime?.close().catch(() => undefined) + await browser.close() + } if (captureSelection.console) { await artifactSession.writeJsonLines("console", "console.jsonl", consoleMessages) } diff --git a/packages/runtime-playground/src/index.ts b/packages/runtime-playground/src/index.ts index 6a74ce646..ec701682c 100644 --- a/packages/runtime-playground/src/index.ts +++ b/packages/runtime-playground/src/index.ts @@ -17,7 +17,7 @@ export { preflightPhpWasmRuntimeAssets, PhpWasmRuntimeAssetIntegrityError, type export { browserPreviewAuthCookieUrls, browserPreviewNetworkPolicySummary, browserPreviewReadinessError, browserPreviewRouting, browserPreviewSecureContextError, browserPreviewTopology, browserPreviewOrigins, resolveBrowserPreviewUrl, type BrowserPreviewNetworkPolicy, type BrowserPreviewTopology } from "./browser-preview-routing.js" export { BROWSER_TRANSPORT_FAULT_CAPABILITIES, applyBrowserTransportFault, createBrowserTransportFaultAdapter, type BrowserTransportFaultAdapter } from "./browser-transport-faults.js" export { PLAYWRIGHT_CLOCK_CONTROL_CAPABILITIES, createBrowserClockController, type BrowserClockController } from "./browser-clock-control.js" -export { PLAYWRIGHT_BROWSER_ENVIRONMENT_CAPABILITIES, applyPlaywrightPageEnvironment, createPlaywrightBrowserEnvironmentContext, resolvePlaywrightBrowserEnvironment, runPlaywrightBrowserEnvironmentMatrix, type BrowserEnvironmentCpuProfile, type BrowserEnvironmentNetworkProfile, type PlaywrightBrowserEnvironmentExecutionInput, type PlaywrightBrowserEnvironmentOptions } from "./browser-environment-matrix.js" +export { PLAYWRIGHT_BROWSER_ENVIRONMENT_CAPABILITIES, applyPlaywrightPageEnvironment, browserEnvironmentCell, createPlaywrightBrowserEnvironmentContext, observePlaywrightBrowserEnvironment, resolvePlaywrightBrowserEnvironment, runPlaywrightBrowserEnvironmentMatrix, type BrowserEnvironmentCpuProfile, type BrowserEnvironmentNetworkProfile, type PlaywrightBrowserEnvironmentExecutionInput, type PlaywrightBrowserEnvironmentOptions, type PlaywrightBrowserEnvironmentRuntime, type PlaywrightBrowserEnvironmentSession } from "./browser-environment-matrix.js" export { WORDPRESS_ADVERSARIAL_ADAPTER_SCHEMA, WORDPRESS_ADVERSARIAL_CAPABILITIES, WORDPRESS_ADVERSARIAL_ORACLES, WORDPRESS_CLOCK_CONTROL_CAPABILITIES, WORDPRESS_HTTP_TRANSPORT_FAULT_CAPABILITIES, createWordPressAdversarialAdapter, evaluateWordPressAdversarialOracles, negotiateWordPressHttpTransportFaults, wordpressAdversarialActionSpec, wordpressHttpFaultConfigurationAction, wordpressNoveltySignals, wordpressSchedulerClockAction, type WordPressAdapterFidelity, type WordPressAdversarialAction, type WordPressAdversarialAdapter, type WordPressAdversarialCapability, type WordPressAdversarialSurface } from "./wordpress-adversarial-adapter.js" export { normalizePreviewReviewerAccess, previewReviewerAccess } from "./preview-reviewer-access.js" export { applyVfsMountSnapshots, materializePlaygroundMountsFromVfs, materializePlaygroundStagedInputs, type HostMountSnapshot, type MountMaterializationResult, type StagedInputMaterializationResult, type VfsMountSnapshot } from "./mount-materialization.js" diff --git a/tests/browser-actions-environment.browser.test.ts b/tests/browser-actions-environment.browser.test.ts new file mode 100644 index 000000000..31ebe2c0b --- /dev/null +++ b/tests/browser-actions-environment.browser.test.ts @@ -0,0 +1,330 @@ +import assert from "node:assert/strict" +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises" +import { createServer } from "node:http" +import { tmpdir } from "node:os" +import { join } from "node:path" +import test from "node:test" + +import type { RuntimeCreateSpec } from "../packages/runtime-core/src/index.js" +import { runBrowserActionsCommand, runBrowserScenarioCommand } from "../packages/runtime-playground/src/browser-actions-runner.js" +import { runBrowserProbeCommand } from "../packages/runtime-playground/src/browser-probe-runner.js" +import type { BrowserArtifact } from "../packages/runtime-playground/src/browser-artifacts.js" +import { isBrowserCommandArtifactError } from "../packages/runtime-playground/src/browser-command-artifact-error.js" +import type { PlaygroundCliServer } from "../packages/runtime-playground/src/preview-server.js" + +const runtimeSpec: RuntimeCreateSpec = { + backend: "wordpress-playground", + environment: {}, + policy: { network: "deny", filesystem: "sandbox", commands: ["wordpress.browser-actions", "wordpress.browser-actions.evaluate", "wordpress.browser-probe", "wordpress.browser-scenario"], secrets: "none", approvals: "never" }, +} + +test("standalone probes use the shared environment context adapter", async () => { + const fixture = await browserFixture() + const artifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-browser-probe-environment-")) + try { + const result = await runBrowserProbeCommand({ + artifactRoot, + runtimeSpec, + server: fixture.server, + spec: { command: "wordpress.browser-probe", args: ["url=/", "device=Pixel 5", "geolocation-latitude=32.7765", "geolocation-longitude=-79.9311", "geolocation-permission=granted", "script=return { touch: navigator.maxTouchPoints > 0, permission: (await navigator.permissions.query({ name: 'geolocation' })).state }", "capture=console"] }, + }) + assert.deepEqual(result.artifact.summary.scriptResult, { touch: true, permission: "granted" }) + assert.equal(result.artifact.summary.viewport?.hasTouch, true) + assert.equal(result.artifact.summary.context?.requested.device, "Pixel 5") + } finally { + await fixture.close() + await rm(artifactRoot, { recursive: true, force: true }) + } +}) + +test("browser actions preserve granted, denied, and prompt environments without leaking between runs", async () => { + const fixture = await browserFixture() + try { + const granted = await runEnvironment(fixture, "granted", `({ permission: (await navigator.permissions.query({ name: "geolocation" })).state, coordinates: await new Promise((resolve) => navigator.geolocation.getCurrentPosition(({ coords }) => resolve([coords.latitude, coords.longitude, coords.accuracy]), ({ code }) => resolve(code), { timeout: 500 })), mobile: /Mobile|Android/.test(navigator.userAgent), touch: navigator.maxTouchPoints > 0 })`, { permission: "granted", coordinates: [32.7765, -79.9311, 9], mobile: true, touch: true }) + assert.deepEqual(granted.summary.environment?.requested.geolocation, { latitude: 32.7765, longitude: -79.9311, accuracy: 9, permission: "granted" }) + assert.equal(granted.summary.viewport?.isMobile, true) + assert.equal(granted.summary.viewport?.hasTouch, true) + + const denied = await runEnvironment(fixture, "denied", `({ permission: (await navigator.permissions.query({ name: "geolocation" })).state, result: await new Promise((resolve) => navigator.geolocation.getCurrentPosition(() => resolve(0), ({ code }) => resolve(code), { timeout: 500 })) })`, { permission: "denied", result: 1 }) + assert.equal(denied.summary.environment?.resolved.geolocation?.permission, "denied") + assert.equal(denied.summary.environment?.observed?.geolocationPermission, "denied") + + const prompt = await runEnvironment(fixture, "prompt", `(await navigator.permissions.query({ name: "geolocation" })).state`, "prompt") + assert.equal(prompt.summary.environment?.resolved.geolocation?.permission, "prompt") + assert.equal(prompt.summary.environment?.observed?.geolocationPermission, "prompt") + assert.deepEqual(prompt.summary.environment?.unsupported, []) + } finally { + await fixture.close() + } +}) + +test("authored scenarios preserve init-script and page state from probe collection into actions", async () => { + const fixture = await browserFixture() + const artifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-browser-scenario-environment-")) + try { + const result = await runBrowserScenarioCommand({ + artifactRoot, + runtimeSpec, + server: fixture.server, + spec: { + command: "wordpress.browser-scenario", + args: [`scenario-json=${JSON.stringify({ + url: "/", + profile: "mobile-chrome", + environment: { userAgent: "Scenario Continuity Agent", permissions: ["notifications"], geolocation: { latitude: 51.5072, longitude: -0.1276, permission: "granted" } }, + captures: ["performance", "steps"], + prePageScript: "globalThis.__scenarioInit = { token: 'probe-owned' }; sessionStorage.setItem('scenario-state', 'preserved')", + steps: [{ kind: "evaluate", expression: `({ permission: (await navigator.permissions.query({ name: "geolocation" })).state, notifications: (await navigator.permissions.query({ name: "notifications" })).state, touch: navigator.maxTouchPoints > 0, userAgent: navigator.userAgent, init: globalThis.__scenarioInit?.token, state: sessionStorage.getItem('scenario-state') })`, assert: { permission: "granted", notifications: "granted", touch: true, userAgent: "Scenario Continuity Agent", init: "probe-owned", state: "preserved" } }], + })}`], + }, + }) + assert.equal(result.artifact.artifactType, "scenario") + assert.equal(result.artifact.summary.environment?.requested.geolocation?.latitude, 51.5072) + assert.equal(result.artifact.summary.environment?.requested.device, "Pixel 5") + assert.equal(result.artifact.summary.environment?.resolved.isMobile, true) + assert.equal(result.artifact.summary.environment?.resolved.hasTouch, true) + assert.equal(result.artifact.summary.environment?.observed?.hasTouch, true) + assert(result.artifact.summary.environment?.inconclusive.includes("browser.environment.device")) + assert.equal(Object.prototype.hasOwnProperty.call(result.artifact.summary.environment ?? {}, "effective"), false) + const output = JSON.parse(result.output) + assert(output.scenario.files.probeSummary) + assert(output.scenario.files.actionSummary) + } finally { + await fixture.close() + await rm(artifactRoot, { recursive: true, force: true }) + } +}) + +test("scenario storage state and auth remain available after probe collection", async () => { + const fixture = await browserFixture() + try { + const storageRoot = await mkdtemp(join(tmpdir(), "wp-codebox-browser-scenario-storage-")) + try { + const storageState = { + cookies: [{ name: "scenario_storage_cookie", value: "ready", domain: "127.0.0.1", path: "/", expires: -1, httpOnly: false, secure: false, sameSite: "Lax" }], + origins: [{ origin: fixture.server.serverUrl, localStorage: [{ name: "scenario-storage", value: "ready" }] }], + } + const stored = await runBrowserScenarioCommand({ + artifactRoot: storageRoot, + runtimeSpec, + server: fixture.server, + spec: { command: "wordpress.browser-scenario", args: [`scenario-json=${JSON.stringify({ url: "/", captures: ["performance", "steps"], steps: [{ kind: "evaluate", expression: "({ storage: localStorage.getItem('scenario-storage'), importedCookie: document.cookie.includes('scenario_storage_cookie=ready'), authCookie: document.cookie.includes('scenario_auth=ready') })", assert: { storage: "ready", importedCookie: true, authCookie: false } }] })}`, `storage-state=${JSON.stringify(storageState)}`] }, + }) + assert.equal(stored.artifact.summary.auth?.mode, "storage-state") + } finally { + await rm(storageRoot, { recursive: true, force: true }) + } + + const authRoot = await mkdtemp(join(tmpdir(), "wp-codebox-browser-scenario-auth-")) + try { + const authenticated = await runBrowserScenarioCommand({ + artifactRoot: authRoot, + runtimeSpec, + server: fixture.server, + runPlaygroundCommand: async () => ({ exitCode: 0, text: JSON.stringify([{ name: "scenario_auth", value: "ready", domain: "127.0.0.1", path: "/", httpOnly: false, secure: false, sameSite: "Lax" }]) }), + spec: { command: "wordpress.browser-scenario", args: [`scenario-json=${JSON.stringify({ url: "/", captures: ["performance", "steps"], auth: "wordpress-admin", authUserId: 7, steps: [{ kind: "evaluate", expression: "({ authCookie: document.cookie.includes('scenario_auth=ready'), importedCookie: document.cookie.includes('scenario_storage_cookie=ready'), storage: localStorage.getItem('scenario-storage') })", assert: { authCookie: true, importedCookie: false, storage: null } }] })}`] }, + }) + assert.equal(authenticated.artifact.summary.auth?.mode, "wordpress-admin") + assert.equal(authenticated.artifact.summary.auth?.userId, 7) + } finally { + await rm(authRoot, { recursive: true, force: true }) + } + + const isolatedStorageRoot = await mkdtemp(join(tmpdir(), "wp-codebox-browser-scenario-storage-isolated-")) + try { + const storageState = { + cookies: [{ name: "scenario_storage_cookie", value: "ready", domain: "127.0.0.1", path: "/", expires: -1, httpOnly: false, secure: false, sameSite: "Lax" }], + origins: [{ origin: fixture.server.serverUrl, localStorage: [{ name: "scenario-storage", value: "ready" }] }], + } + await runBrowserScenarioCommand({ + artifactRoot: isolatedStorageRoot, + runtimeSpec, + server: fixture.server, + spec: { command: "wordpress.browser-scenario", args: [`scenario-json=${JSON.stringify({ url: "/", captures: ["performance", "steps"], steps: [{ kind: "evaluate", expression: "({ storage: localStorage.getItem('scenario-storage'), importedCookie: document.cookie.includes('scenario_storage_cookie=ready'), authCookie: document.cookie.includes('scenario_auth=ready') })", assert: { storage: "ready", importedCookie: true, authCookie: false } }] })}`, `storage-state=${JSON.stringify(storageState)}`] }, + }) + } finally { + await rm(isolatedStorageRoot, { recursive: true, force: true }) + } + } finally { + await fixture.close() + } +}) + +test("scenario declarations override command arguments and reject unknown profiles", async () => { + const fixture = await browserFixture() + const artifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-browser-scenario-precedence-")) + try { + const result = await runBrowserScenarioCommand({ + artifactRoot, + runtimeSpec, + server: fixture.server, + spec: { + command: "wordpress.browser-scenario", + args: [ + `scenario-json=${JSON.stringify({ url: "/", profile: "mobile-chrome", viewport: "800x600", environment: { geolocation: { latitude: 51.5072, longitude: -0.1276, permission: "granted" } }, steps: [{ kind: "evaluate", expression: "({ width: innerWidth, height: innerHeight, permission: (await navigator.permissions.query({ name: 'geolocation' })).state })", assert: { width: 800, height: 600, permission: "granted" } }] })}`, + "viewport=320x640", + "device=Desktop Chrome", + "geolocation-latitude=32.7765", + "geolocation-longitude=-79.9311", + "geolocation-permission=denied", + ], + }, + }) + assert.deepEqual(result.artifact.summary.environment?.requested.viewport, { width: 800, height: 600 }) + assert.equal(result.artifact.summary.environment?.requested.device, "Pixel 5") + assert.equal(result.artifact.summary.environment?.requested.geolocation?.latitude, 51.5072) + assert.equal(result.artifact.summary.environment?.observed?.geolocationPermission, "granted") + + const unknownProfileRoot = await mkdtemp(join(tmpdir(), "wp-codebox-browser-scenario-profile-")) + try { + await assert.rejects(runBrowserScenarioCommand({ + artifactRoot: unknownProfileRoot, + runtimeSpec, + server: fixture.server, + spec: { command: "wordpress.browser-scenario", args: [`scenario-json=${JSON.stringify({ url: "/", profile: "unknown-mobile" })}`] }, + }), /unknown profile: unknown-mobile/) + } finally { + await rm(unknownProfileRoot, { recursive: true, force: true }) + } + } finally { + await fixture.close() + await rm(artifactRoot, { recursive: true, force: true }) + } +}) + +test("unsupported action environments fail with structured requested and unsupported evidence", async () => { + const fixture = await browserFixture() + const artifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-browser-actions-unsupported-")) + try { + await assert.rejects(runBrowserActionsCommand({ + artifactRoot, + runtimeSpec, + server: fixture.server, + spec: { command: "wordpress.browser-actions", args: ["url=/", "device=Unknown Device", "capture=steps"] }, + }), (error: unknown) => { + assert(isBrowserCommandArtifactError(error)) + assert.equal(error.artifact.summary.environment?.requested.device, "Unknown Device") + assert.deepEqual(error.artifact.summary.environment?.unsupported, ["browser.environment.device"]) + assert.equal(error.artifact.summary.environment?.capabilities.find(({ id }) => id === "browser.environment.device")?.fidelity, "unsupported") + return true + }) + } finally { + await fixture.close() + await rm(artifactRoot, { recursive: true, force: true }) + } +}) + +test("adaptive actions retain their declared environment through routed preview origins", async () => { + const fixture = await browserFixture() + const routedHost = "browser-actions.test" + try { + const artifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-browser-actions-adaptive-")) + try { + const result = await runBrowserActionsCommand({ + artifactRoot, + runtimeSpec, + server: fixture.server, + spec: { + command: "wordpress.browser-actions", + args: [ + `adaptive-exploration-json=${JSON.stringify({ schema: "wp-codebox/browser-adaptive-exploration/v1", startUrl: `http://${routedHost}:${fixture.port}/`, seed: "environment", environment: { device: "Pixel 5" }, budgets: { maxStates: 2, maxTransitions: 2, maxDepth: 1, maxDurationMs: 2_000, maxArtifactBytes: 100_000 }, failOnFinding: false })}`, + `route-host=${routedHost}`, + "capture=steps", + ], + }, + }) + assert.equal(new URL(result.artifact.requestedUrl).hostname, routedHost) + assert.equal(result.artifact.summary.viewport?.isMobile, true) + assert.equal(result.artifact.summary.viewport?.hasTouch, true) + assert.equal(result.artifact.summary.environment?.resolved.device, "Pixel 5") + assert.equal(result.artifact.summary.adaptiveExploration?.schema, "wp-codebox/browser-adaptive-exploration/v1") + const adaptiveArtifact = JSON.parse(await readFile(join(artifactRoot, "files/browser/adaptive-exploration.json"), "utf8")) + assert.equal(adaptiveArtifact.result.replay.environment.device, "Pixel 5") + assert.equal(adaptiveArtifact.result.replay.environmentDigest, adaptiveArtifact.contract.environmentDigest) + + await assert.rejects(runBrowserActionsCommand({ + artifactRoot, + runtimeSpec, + server: fixture.server, + spec: { + command: "wordpress.browser-actions", + args: [ + `adaptive-exploration-json=${JSON.stringify({ schema: "wp-codebox/browser-adaptive-exploration/v1", startUrl: "/", environment: { locale: "en-GB" } })}`, + "device=Pixel 5", + ], + }, + }), /environment cannot be combined with outer browser environment arguments/) + } finally { + await rm(artifactRoot, { recursive: true, force: true }) + } + } finally { + await fixture.close() + } +}) + +test("scenario steps-json file payloads execute instead of being dropped", async () => { + const fixture = await browserFixture() + const artifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-browser-scenario-steps-file-")) + try { + const stepsPath = join(artifactRoot, "steps.json") + await writeFile(stepsPath, JSON.stringify([{ kind: "evaluate", expression: "document.title", assert: "" }])) + const result = await runBrowserScenarioCommand({ + artifactRoot, + runtimeSpec, + server: fixture.server, + spec: { command: "wordpress.browser-scenario", args: ["url=/", `steps-json=@${stepsPath}`, "capture=steps"] }, + }) + assert.equal(result.artifact.summary.actions, 2) + } finally { + await fixture.close() + await rm(artifactRoot, { recursive: true, force: true }) + } +}) + +async function runEnvironment(fixture: Awaited>, permission: "granted" | "denied" | "prompt", expression: string, expected: unknown): Promise { + const artifactRoot = await mkdtemp(join(tmpdir(), `wp-codebox-browser-actions-${permission}-`)) + try { + const result = await runBrowserActionsCommand({ + artifactRoot, + runtimeSpec, + server: fixture.server, + spec: { + command: "wordpress.browser-actions", + args: [ + `url=http://localhost:${fixture.port}/`, + `steps-json=${JSON.stringify([{ kind: "evaluate", expression, assert: expected }])}`, + "device=Pixel 5", + "geolocation-latitude=32.7765", + "geolocation-longitude=-79.9311", + "geolocation-accuracy=9", + `geolocation-permission=${permission}`, + "route-host=localhost", + "capture=steps", + ], + }, + }) + return result.artifact + } finally { + await rm(artifactRoot, { recursive: true, force: true }) + } +} + +async function browserFixture(): Promise<{ port: number; server: PlaygroundCliServer; close(): Promise }> { + const httpServer = createServer((_request, response) => { + response.setHeader("content-type", "text/html") + response.end("") + }) + await new Promise((resolve) => httpServer.listen(0, "127.0.0.1", resolve)) + const address = httpServer.address() + assert(address && typeof address === "object") + const server = { + serverUrl: `http://127.0.0.1:${address.port}`, + playground: { async run() { return { text: "", exitCode: 0 } } }, + async [Symbol.asyncDispose]() {}, + } satisfies PlaygroundCliServer + return { + port: address.port, + server, + close: () => new Promise((resolve, reject) => httpServer.close((error) => error ? reject(error) : resolve())), + } +} diff --git a/tests/browser-adaptive-exploration.test.ts b/tests/browser-adaptive-exploration.test.ts index b3a0fdcf7..365bc36f0 100644 --- a/tests/browser-adaptive-exploration.test.ts +++ b/tests/browser-adaptive-exploration.test.ts @@ -17,6 +17,10 @@ import { executeBrowserInteractionStep } from "../packages/runtime-playground/sr import { browserPreviewTopology, routeBrowserPreviewContextNetwork } from "../packages/runtime-playground/src/browser-preview-routing.js" import { closeHttpServer, listenLocalHttpServer } from "../packages/runtime-playground/src/preview-server.js" +function environmentBoundOracleFingerprint(value: string): string { + return browserAdaptiveDigest("oracle", { environmentDigest: browserAdaptiveExplorationContract({}).environmentDigest, value }) +} + const modalFixture = ` @@ -317,6 +321,19 @@ test("rediscovery finds, minimizes, and replays a defect revealed by a dynamic m assert.deepEqual(finding.replay.actions, finding.minimizedPath) assert.equal(finding.replay.expectedFingerprint, finding.fingerprint) assert.equal(finding.replay.expectedStateDigest, finding.stateDigest) + assert.deepEqual(finding.replay.environment, {}) + assert.equal(finding.replay.environmentDigest, run.contract.environmentDigest) + + const mobile = await runFixture(modalFixture, { + seed: "modal-seed", + environment: { viewport: { width: 390, height: 844 }, isMobile: true, hasTouch: true, geolocation: { latitude: 32.7765, longitude: -79.9311, permission: "granted" } }, + budgets: { maxActions: 80, maxStates: 24, maxTransitions: 80, maxDurationMs: 30_000, maxArtifactBytes: 2_000_000, maxErrors: 10 }, + actionFamilies: ["click", "fill", "submit", "repeat", "double-submit"], + }) + assert.notEqual(mobile.result.states[0]?.digest, run.result.states[0]?.digest) + assert.notEqual(mobile.result.findings[0]?.fingerprint, finding.fingerprint) + assert.deepEqual(mobile.result.replay.environment, mobile.contract.environment) + assert.equal(mobile.result.findings[0]?.replay.environmentDigest, mobile.contract.environmentDigest) }) test("bounded stabilization captures delayed conditional fields and route state", async () => { @@ -519,7 +536,7 @@ test("URL-less policy-block console records correlate before promotion and prese const message = urlLess.result.transitions[0]?.observations.consoleErrors[0] assert(message) assert.equal(urlLess.result.transitions[0]?.observations.oracleFingerprints.length, 1) - assert.equal(urlLess.result.findings[0]?.fingerprint, browserAdaptiveDigest("oracle", message)) + assert.equal(urlLess.result.findings[0]?.fingerprint, environmentBoundOracleFingerprint(message)) assert.equal(urlLess.result.findings[0]?.fingerprint, attributed.result.findings[0]?.fingerprint, "removing console location must not change the historical console fingerprint") const mixed = await runNetworkOracleFixture("block", "mixed", {}, true) @@ -531,7 +548,7 @@ test("URL-less policy-block console records correlate before promotion and prese test("same-URL product errors require a matching failure token before policy correlation", async () => { const productMessage = "same-URL product defect" - const productFingerprint = browserAdaptiveDigest("oracle", productMessage) + const productFingerprint = environmentBoundOracleFingerprint(productMessage) const evidence = await runNetworkOracleFixture("block", "blocked", {}, false, true) assert.deepEqual(evidence.result.transitions[0]?.observations.oracleFingerprints, [productFingerprint]) assert.equal(evidence.result.findings[0]?.fingerprint, productFingerprint) @@ -542,7 +559,7 @@ test("same-URL product errors require a matching failure token before policy cor assert(blockMessage) assert.equal(fingerprints.length, 2) assert(fingerprints.includes(productFingerprint)) - assert(fingerprints.includes(browserAdaptiveDigest("oracle", blockMessage))) + assert(fingerprints.includes(environmentBoundOracleFingerprint(blockMessage))) assert.equal(finding.result.transitions[0]?.observations.networkFailureSummary?.policyBlocks, 1) }) diff --git a/tests/browser-environment-matrix.browser.test.ts b/tests/browser-environment-matrix.browser.test.ts index a3e423101..b3ae91399 100644 --- a/tests/browser-environment-matrix.browser.test.ts +++ b/tests/browser-environment-matrix.browser.test.ts @@ -122,6 +122,7 @@ test("real browser contexts apply and isolate granted, denied, and prompt geoloc { cell: "granted", requested: "granted", permission: "granted", latitude: 32.7765, longitude: -79.9311, accuracy: 9 }, { cell: "prompt", requested: "prompt", permission: "prompt" }, ]) + assert.deepEqual(report.cells.map((cell) => cell.effective.permissions), [[], ["geolocation"], []]) const cells = matrix.dimensions[0]!.values.slice(1).map((value, index) => ({ id: value.id, index, seed: value.id, selections: { location: value.id }, requested: value.environment, requiredCapabilities: [], optionalCapabilities: [] })) const parallel = await Promise.all(cells.map(async (cell) => { diff --git a/tests/browser-multi-actor-scenario.test.ts b/tests/browser-multi-actor-scenario.test.ts index 05384e9d8..fdfd17f45 100644 --- a/tests/browser-multi-actor-scenario.test.ts +++ b/tests/browser-multi-actor-scenario.test.ts @@ -1,7 +1,13 @@ import assert from "node:assert/strict" +import { mkdtemp, rm } from "node:fs/promises" +import { createServer } from "node:http" +import { tmpdir } from "node:os" +import { join } from "node:path" import { BROWSER_MULTI_ACTOR_SCENARIO_SCHEMA, type BrowserMultiActorScenario } from "../packages/runtime-core/src/browser-multi-actor-scenario-contracts.js" +import type { RuntimeCreateSpec } from "../packages/runtime-core/src/runtime-contracts.js" import { BrowserMultiActorScenarioError, runBrowserMultiActorScenario, type BrowserMultiActorClient } from "../packages/runtime-playground/src/browser-multi-actor-scenario.js" -import { navigateBrowserMultiActorPages } from "../packages/runtime-playground/src/browser-multi-actor-scenario-runner.js" +import { navigateBrowserMultiActorPages, runBrowserMultiActorScenarioCommand } from "../packages/runtime-playground/src/browser-multi-actor-scenario-runner.js" +import type { PlaygroundCliServer } from "../packages/runtime-playground/src/preview-server.js" const navigationStarted: string[] = [] let releaseNavigation!: () => void @@ -95,4 +101,55 @@ await assert.rejects( runBrowserMultiActorScenario({ ...scenario, requestGates: [{ ...scenario.requestGates![0]!, occurrence: 0 }] }, clients), /positive occurrence/, ) + +const environmentServer = createServer((_request, response) => response.end("
actors
")) +await new Promise((resolve) => environmentServer.listen(0, "127.0.0.1", resolve)) +const environmentAddress = environmentServer.address() +assert(environmentAddress && typeof environmentAddress === "object") +const environmentArtifacts = await mkdtemp(join(tmpdir(), "wp-codebox-multi-actor-environment-")) +const environmentRuntimeSpec = { + backend: "wordpress-playground", + environment: {}, + policy: { network: "deny", filesystem: "sandbox", commands: ["wordpress.browser-scenario"], secrets: "none", approvals: "never" }, + metadata: { recipe: { inputs: { fixtureUsers: [{ name: "author", userId: 11, role: "author" }, { name: "reviewer", userId: 12, role: "editor" }], userSessions: [{ name: "author-session", user: "author" }, { name: "reviewer-session", user: "reviewer" }] } } }, +} as RuntimeCreateSpec +const environmentCliServer = { + serverUrl: `http://127.0.0.1:${environmentAddress.port}`, + playground: { async run() { return { text: "", exitCode: 0 } } }, + async [Symbol.asyncDispose]() {}, +} satisfies PlaygroundCliServer +try { + const result = await runBrowserMultiActorScenarioCommand({ + artifactRoot: environmentArtifacts, + runtimeSpec: environmentRuntimeSpec, + server: environmentCliServer, + runPlaygroundCommand: async (_command, _server, _options) => ({ exitCode: 0, text: JSON.stringify([{ name: "actor_auth", value: "ready", domain: "127.0.0.1", path: "/", httpOnly: false, sameSite: "Lax" }]) }), + scenario: { + schema: BROWSER_MULTI_ACTOR_SCENARIO_SCHEMA, + seed: "actor-environments", + url: "/", + environment: { device: "Pixel 5", geolocation: { latitude: 32.7765, longitude: -79.9311, permission: "granted" } }, + browserArgs: [], + actors: [{ name: "author", userSession: "author-session" }, { name: "reviewer", userSession: "reviewer-session" }], + actions: [ + { id: "author-environment", actor: "author", step: { kind: "evaluate", expression: "const prior = localStorage.getItem('actor'); localStorage.setItem('actor', 'author'); return { prior, actor: localStorage.getItem('actor'), touch: navigator.maxTouchPoints > 0, permission: (await navigator.permissions.query({ name: 'geolocation' })).state }", assert: { prior: null, actor: "author", touch: true, permission: "granted" } } }, + { id: "reviewer-environment", actor: "reviewer", step: { kind: "evaluate", expression: "const prior = localStorage.getItem('actor'); localStorage.setItem('actor', 'reviewer'); return { prior, actor: localStorage.getItem('actor'), touch: navigator.maxTouchPoints > 0, permission: (await navigator.permissions.query({ name: 'geolocation' })).state }", assert: { prior: null, actor: "reviewer", touch: true, permission: "granted" } } }, + ], + }, + }) + const output = JSON.parse(result.output) + assert.equal(output.scenario.actors.author.environment.observed.hasTouch, true) + assert.equal(output.scenario.actors.reviewer.environment.observed.hasTouch, true) + assert.notStrictEqual(output.scenario.actors.author.environment, output.scenario.actors.reviewer.environment) + + await assert.rejects(runBrowserMultiActorScenarioCommand({ + artifactRoot: environmentArtifacts, + runtimeSpec: environmentRuntimeSpec, + server: environmentCliServer, + scenario: { schema: BROWSER_MULTI_ACTOR_SCENARIO_SCHEMA, seed: "unsupported", url: "/", environment: { device: "Unknown Device" }, browserArgs: [], actors: [], actions: [] }, + }), /browser environment is unsupported: browser.environment.device/) +} finally { + environmentServer.close() + await rm(environmentArtifacts, { recursive: true, force: true }) +} console.log("multi-actor browser scenarios ok") diff --git a/tests/browser-recipe-file-payloads.integration.test.ts b/tests/browser-recipe-file-payloads.integration.test.ts new file mode 100644 index 000000000..357472849 --- /dev/null +++ b/tests/browser-recipe-file-payloads.integration.test.ts @@ -0,0 +1,76 @@ +import assert from "node:assert/strict" +import { execFile } from "node:child_process" +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import { dirname, join, resolve } from "node:path" +import { fileURLToPath } from "node:url" +import { promisify } from "node:util" + +const execFileAsync = promisify(execFile) +const repositoryRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..") +const root = await mkdtemp(join(tmpdir(), "wp-codebox-browser-recipe-files-")) +const recipeDirectory = join(root, "recipe") +const invocationDirectory = join(root, "invocation") +const cli = join(repositoryRoot, "packages/cli/dist/index.js") + +try { + await Promise.all([mkdir(recipeDirectory), mkdir(invocationDirectory)]) + await writeFile(join(recipeDirectory, "action-steps.json"), JSON.stringify([{ kind: "evaluate", expression: "document.documentElement.dataset.actionRecipe = 'ready'", assert: "ready" }])) + await writeFile(join(recipeDirectory, "scenario.json"), JSON.stringify({ url: "about:blank", captures: ["steps"], steps: [{ kind: "evaluate", expression: "document.documentElement.dataset.scenarioRecipe = 'ready'", assert: "ready" }] })) + await writeFile(join(recipeDirectory, "scenario-steps.json"), JSON.stringify([{ kind: "evaluate", expression: "document.documentElement.dataset.scenarioStepsRecipe = 'ready'", assert: "ready" }])) + await writeFile(join(recipeDirectory, "valid-recipe.json"), JSON.stringify({ + schema: "wp-codebox/workspace-recipe/v1", + runtime: { backend: "wordpress-playground", wp: "latest", blueprint: { steps: [] } }, + workflow: { steps: [ + { command: "wordpress.browser-actions", args: ["url=about:blank", "steps-json=@action-steps.json", "capture=steps"] }, + { command: "wordpress.browser-scenario", args: ["scenario-json=@scenario.json"] }, + { command: "wordpress.browser-scenario", args: ["url=about:blank", "steps-json=@scenario-steps.json", "capture=steps"] }, + ] }, + })) + + const validDryRun = await execFileAsync(process.execPath, [cli, "recipe-run", "--recipe", join(recipeDirectory, "valid-recipe.json"), "--dry-run", "--json"], { + cwd: invocationDirectory, + timeout: 120_000, + maxBuffer: 4 * 1024 * 1024, + }) + const validDryRunOutput = JSON.parse(validDryRun.stdout) + assert.equal(validDryRunOutput.plan.policy.commands.includes("wordpress.browser-actions.evaluate"), true) + + const valid = await execFileAsync(process.execPath, [cli, "recipe-run", "--recipe", join(recipeDirectory, "valid-recipe.json"), "--artifacts", join(root, "valid-artifacts"), "--json"], { + cwd: invocationDirectory, + timeout: 300_000, + maxBuffer: 4 * 1024 * 1024, + }) + const validOutput = JSON.parse(valid.stdout) + assert.equal(validOutput.success, true, valid.stdout) + assert.equal(validOutput.executions.length, 3) + + await writeFile(join(recipeDirectory, "malformed.json"), "{") + await writeFile(join(recipeDirectory, "malformed-recipe.json"), JSON.stringify({ + schema: "wp-codebox/workspace-recipe/v1", + workflow: { steps: [ + { command: "wordpress.browser-actions", args: ["url=/", "steps-json=@malformed.json"] }, + { command: "wordpress.browser-scenario", args: ["scenario-json=@malformed.json"] }, + { command: "wordpress.browser-scenario", args: ["url=/", "steps-json=@malformed.json"] }, + ] }, + })) + + let malformedStdout = "" + try { + await execFileAsync(process.execPath, [cli, "recipe-run", "--recipe", join(recipeDirectory, "malformed-recipe.json"), "--dry-run", "--json"], { + cwd: invocationDirectory, + timeout: 120_000, + maxBuffer: 4 * 1024 * 1024, + }) + assert.fail("Malformed file-backed browser payloads should fail recipe validation") + } catch (error) { + malformedStdout = String((error as { stdout?: string }).stdout ?? "") + } + const malformedOutput = JSON.parse(malformedStdout) + assert.equal(malformedOutput.success, false) + assert.deepEqual(malformedOutput.validation.issues.map((issue: { code: string }) => issue.code), ["invalid-steps-json", "invalid-scenario-json", "invalid-steps-json"]) +} finally { + await rm(root, { recursive: true, force: true }) +} + +console.log("browser recipe file payload integration passed") diff --git a/tests/recipe-validation-descriptors.test.ts b/tests/recipe-validation-descriptors.test.ts index 9fc40ddd8..1c293efd3 100644 --- a/tests/recipe-validation-descriptors.test.ts +++ b/tests/recipe-validation-descriptors.test.ts @@ -1,7 +1,8 @@ import assert from "node:assert/strict" +import { writeFile } from "node:fs/promises" import { join } from "node:path" -import { validateWorkspaceRecipeSemantics } from "../packages/cli/src/recipe-validation.js" +import { recipePolicy, validateWorkspaceRecipeSemantics } from "../packages/cli/src/recipe-validation.js" import type { WorkspaceRecipe } from "../packages/runtime-core/src/index.js" import { withTempDir } from "../scripts/test-kit.js" @@ -50,12 +51,68 @@ await withTempDir("wp-codebox-recipe-geolocation-validation-", async (recipeDire { command: "wordpress.browser-probe", args: ["url=/", "geolocation-latitude=NaN", "geolocation-longitude=181", "geolocation-accuracy=-1", "geolocation-permission=maybe"] }, { command: "wordpress.browser-probe", args: ["url=/", "geolocation-latitude=32.7765", "geolocation-permission=granted"] }, { command: "wordpress.browser-probe", args: ["url=/", "geolocation-latitude=32.7765", "geolocation-longitude=-79.9311", "geolocation-accuracy=8", "geolocation-permission=default"] }, + { command: "wordpress.browser-actions", args: ["url=/", "geolocation-latitude=32.7765", "geolocation-permission=granted", "is-mobile=maybe", "has-touch=true"] }, + { command: "wordpress.browser-scenario", args: ["url=/", "browser-environment-json={\"isMobile\":\"yes\"}"] }, + { command: "wordpress.browser-actions", args: ["url=/", "browser-environment-json={}", "steps-json={"] }, + { command: "wordpress.browser-scenario", args: ["url=/", "browser-environment-json={}", "steps-json={"] }, + { command: "wordpress.browser-scenario", args: [`scenario-json=${JSON.stringify({ url: "/", profile: "unknown-mobile" })}`] }, ] }, } const issues = await validateWorkspaceRecipeSemantics(recipe, join(recipeDirectory, "recipe.json")) assert.deepEqual(issues.filter(({ path }) => path === "$.workflow.steps[0].args").map(({ code }) => code), ["invalid-geolocation-latitude", "invalid-geolocation-longitude", "invalid-geolocation-accuracy", "invalid-geolocation-permission"]) assert.deepEqual(issues.filter(({ path }) => path === "$.workflow.steps[1].args").map(({ code }) => code), ["incomplete-geolocation", "incomplete-geolocation"]) assert.deepEqual(issues.filter(({ path }) => path === "$.workflow.steps[2].args"), []) + assert.deepEqual(issues.filter(({ path }) => path === "$.workflow.steps[3].args").map(({ code }) => code), ["invalid-is-mobile", "incomplete-geolocation", "incomplete-geolocation"]) + assert.deepEqual(issues.filter(({ path }) => path === "$.workflow.steps[4].args").map(({ code }) => code), ["invalid-browser-environment"]) + assert.deepEqual(issues.filter(({ path }) => path === "$.workflow.steps[5].args").map(({ code }) => code), ["invalid-steps-json"]) + assert.deepEqual(issues.filter(({ path }) => path === "$.workflow.steps[6].args").map(({ code }) => code), ["invalid-steps-json"]) + assert.deepEqual(issues.filter(({ path }) => path === "$.workflow.steps[7].args").map(({ code }) => code), ["invalid-profile"]) +}) + +await withTempDir("wp-codebox-recipe-browser-environment-file-", async (recipeDirectory) => { + await writeFile(join(recipeDirectory, "valid-environment.json"), JSON.stringify({ isMobile: true, hasTouch: true })) + await writeFile(join(recipeDirectory, "invalid-environment.json"), "{") + const recipe: WorkspaceRecipe = { + schema: "wp-codebox/workspace-recipe/v1", + workflow: { steps: [ + { command: "wordpress.browser-actions", args: ["url=/", "browser-environment-json=@valid-environment.json"] }, + { command: "wordpress.browser-actions", args: ["url=/", "browser-environment-json=@invalid-environment.json"] }, + { command: "wordpress.browser-scenario", args: ["url=/", "browser-environment-json=@missing-environment.json"] }, + ] }, + } + const issues = await validateWorkspaceRecipeSemantics(recipe, join(recipeDirectory, "recipe.json")) + assert.deepEqual(issues.filter(({ path }) => path === "$.workflow.steps[0].args"), []) + assert.deepEqual(issues.filter(({ path }) => path === "$.workflow.steps[1].args").map(({ code }) => code), ["invalid-browser-environment"]) + assert.deepEqual(issues.filter(({ path }) => path === "$.workflow.steps[2].args").map(({ code }) => code), ["invalid-browser-environment"]) +}) + +await withTempDir("wp-codebox-recipe-browser-payload-files-", async (recipeDirectory) => { + await writeFile(join(recipeDirectory, "action-steps.json"), JSON.stringify([{ kind: "evaluate", expression: "document.title" }])) + await writeFile(join(recipeDirectory, "scenario.json"), JSON.stringify({ url: "/", steps: [{ kind: "evaluate", expression: "location.href" }] })) + await writeFile(join(recipeDirectory, "scenario-steps.json"), JSON.stringify([{ kind: "evaluate", expression: "document.body.dataset.ready" }])) + await writeFile(join(recipeDirectory, "malformed.json"), "{") + const recipe: WorkspaceRecipe = { + schema: "wp-codebox/workspace-recipe/v1", + workflow: { steps: [ + { command: "wordpress.browser-actions", args: ["url=/", "steps-json=@action-steps.json"] }, + { command: "wordpress.browser-scenario", args: ["scenario-json=@scenario.json"] }, + { command: "wordpress.browser-scenario", args: ["url=/", "steps-json=@scenario-steps.json"] }, + { command: "wordpress.browser-actions", args: ["url=/", "steps-json=@malformed.json"] }, + { command: "wordpress.browser-scenario", args: ["scenario-json=@malformed.json"] }, + { command: "wordpress.browser-scenario", args: ["url=/", "steps-json=@missing.json"] }, + ] }, + } + const issues = await validateWorkspaceRecipeSemantics(recipe, join(recipeDirectory, "recipe.json")) + assert.deepEqual(issues.filter(({ path }) => path === "$.workflow.steps[0].args"), []) + assert.deepEqual(issues.filter(({ path }) => path === "$.workflow.steps[1].args"), []) + assert.deepEqual(issues.filter(({ path }) => path === "$.workflow.steps[2].args"), []) + assert.deepEqual(issues.filter(({ path }) => path === "$.workflow.steps[3].args").map(({ code }) => code), ["invalid-steps-json"]) + assert.deepEqual(issues.filter(({ path }) => path === "$.workflow.steps[4].args").map(({ code }) => code), ["invalid-scenario-json"]) + assert.deepEqual(issues.filter(({ path }) => path === "$.workflow.steps[5].args").map(({ code }) => code), ["invalid-steps-json"]) + assert.equal(recipePolicy(recipe, recipeDirectory).commands.includes("wordpress.browser-actions.evaluate"), true) + for (const step of recipe.workflow.steps.slice(0, 3)) { + assert.equal(recipePolicy({ schema: recipe.schema, workflow: { steps: [step] } }, recipeDirectory).commands.includes("wordpress.browser-actions.evaluate"), true) + } }) console.log("recipe validation descriptors ok")