Require real scoped PHPUnit execution

Author: chubes4

packages/cli/src/recipe-validation.ts

@@ -743,11 +743,13 @@ function validateRecipeRuntimeServices(recipe: WorkspaceRecipe, addIssue: (code:
     if (ids.has(service.id)) addIssue("duplicate-runtime-service-id", `${path}.id`, `Runtime service ids must be unique: ${service.id}`)
     ids.add(service.id)
     if (service.kind !== "mysql") addIssue("unsupported-runtime-service-kind", `${path}.kind`, `Unsupported managed runtime service kind: ${service.kind}`)
-    for (const [output, name] of Object.entries(service.outputs)) {
+    for (const [output, names] of Object.entries(service.outputs)) {
       if (!/^(host|port|username|password|database)$/.test(output)) addIssue("unknown-runtime-service-output", `${path}.outputs.${output}`, `Unsupported ${service.kind} service output: ${output}`)
-      if (!/^[A-Z_][A-Z0-9_]*$/.test(name)) addIssue("invalid-runtime-service-env", `${path}.outputs.${output}`, "Runtime service environment variable names must match /^[A-Z_][A-Z0-9_]*$/.")
-      if (environment.has(name)) addIssue("duplicate-runtime-service-env", `${path}.outputs.${output}`, `Runtime service output environment variable is already declared: ${name}`)
-      environment.add(name)
+      for (const name of Array.isArray(names) ? names : [names]) {
+        if (!/^[A-Z_][A-Z0-9_]*$/.test(name)) addIssue("invalid-runtime-service-env", `${path}.outputs.${output}`, "Runtime service environment variable names must match /^[A-Z_][A-Z0-9_]*$/.")
+        if (environment.has(name)) addIssue("duplicate-runtime-service-env", `${path}.outputs.${output}`, `Runtime service output environment variable is already declared: ${name}`)
+        environment.add(name)
+      }
     }
   }
 }

packages/cli/src/runtime-services.ts

@@ -61,7 +61,7 @@ const defaultDependencies: RuntimeServiceDependencies = {
   randomBytes,
 }
 
-export function runtimeServicePlan(services: WorkspaceRecipeRuntimeService[]): Array<{ id: string; kind: string; provider: string; version: string; bind: "loopback"; port: "ephemeral"; persistentVolume: false; configuration?: WorkspaceRecipeRuntimeService["configuration"]; outputs: Record<string, string> }> {
+export function runtimeServicePlan(services: WorkspaceRecipeRuntimeService[]): Array<{ id: string; kind: string; provider: string; version: string; bind: "loopback"; port: "ephemeral"; persistentVolume: false; configuration?: WorkspaceRecipeRuntimeService["configuration"]; outputs: Record<string, string | string[]> }> {
   return services.map((service) => {
     const provider = runtimeServiceProvider(service.kind)
     return { id: service.id, kind: service.kind, provider: provider.name, version: provider.version(service), bind: "loopback", port: "ephemeral", persistentVolume: false, ...(service.configuration ? { configuration: service.configuration } : {}), outputs: service.outputs }
@@ -177,7 +177,7 @@ async function provisionMysqlDockerService(service: WorkspaceRecipeRuntimeServic
     evidence.readiness = "ready"
     evidence.lifecycle = "provisioned"
     const values: Record<string, string> = { host: "127.0.0.1", port: String(port), username: "runtime", password, database: "runtime" }
-    return { env: Object.fromEntries(Object.entries(service.outputs).map(([output, name]) => [name, values[output] ?? ""])), evidence, async release() { await releaseService(container, evidence, dependencies) } }
+    return { env: Object.fromEntries(Object.entries(service.outputs).flatMap(([output, names]) => (Array.isArray(names) ? names : [names]).map((name) => [name, values[output] ?? ""]))), evidence, async release() { await releaseService(container, evidence, dependencies) } }
   } catch (error) {
     evidence.readiness = "failed"
     evidence.lifecycle = "failed"

packages/runtime-core/src/recipe-schema.ts

@@ -930,7 +930,12 @@ export function createWorkspaceRecipeJsonSchema(options: WorkspaceRecipeJsonSche
           outputs: {
             type: "object",
             minProperties: 1,
-            additionalProperties: { type: "string", pattern: "^[A-Z_][A-Z0-9_]*$" },
+            additionalProperties: {
+              anyOf: [
+                { type: "string", pattern: "^[A-Z_][A-Z0-9_]*$" },
+                { type: "array", minItems: 1, uniqueItems: true, items: { type: "string", pattern: "^[A-Z_][A-Z0-9_]*$" } },
+              ],
+            },
           },
         },
       },

packages/runtime-core/src/runtime-contracts.ts

@@ -142,8 +142,8 @@ export interface WorkspaceRecipeRuntimeService {
     rootAuthentication?: "generated-password" | "empty-password"
     foreignKeyTargetPolicy?: "unique-only" | "indexed"
   }
-  /** Explicit map from a provider output (for example `port`) to a runtime env name. */
-  outputs: Record<string, string>
+  /** Explicit map from a provider output (for example `port`) to one or more runtime env names. */
+  outputs: Record<string, string | string[]>
 }
 
 export interface WorkspaceRecipeRuntimeStack {

packages/runtime-playground/src/phpunit-command-handlers.ts

@@ -97,7 +97,7 @@ function phpunitConfigDiscoveryPhp(options: PhpunitConfigDiscoveryPhpOptions): s
   const suffixAssignment = options.replaceDefaultMatchers ? "$suffixes = $config_suffixes;" : "$suffixes = array_merge($suffixes, $config_suffixes);"
   const prefixAssignment = options.replaceDefaultMatchers ? "$prefixes = $config_prefixes;" : "$prefixes = array_merge($prefixes, $config_prefixes);"
 
-  return `function ${options.functionName}($xml_path, $test_dir_default) {
+  return `function ${options.functionName}($xml_path, $test_dir_default, $requested_suites = array()) {
     $directories = array($test_dir_default);
     $suffixes = array('Test.php');
     $prefixes = array('test-');
@@ -118,11 +118,23 @@ function phpunitConfigDiscoveryPhp(options: PhpunitConfigDiscoveryPhpOptions): s
         $first = $errors ? trim($errors[0]->message) : 'unknown';
         throw new RuntimeException('PHPUnit config could not be parsed at ' . $xml_path . ': ' . $first);
     }
+    $suite_nodes = $xml->xpath('//testsuite') ?: array();
+    if (!empty($requested_suites)) {
+        $requested_lookup = array_fill_keys(array_map('strval', $requested_suites), true);
+        $suite_nodes = array_values(array_filter($suite_nodes, static function($suite) use ($requested_lookup) {
+            return isset($requested_lookup[(string) ($suite['name'] ?? '')]);
+        }));
+        if (empty($suite_nodes)) {
+            throw new RuntimeException('Requested PHPUnit testsuite was not found in config: ' . implode(',', array_keys($requested_lookup)));
+        }
+    }
+    $directories = array();
     $base = ${options.basePathExpression};
     $config_dirs = array();
     $config_suffixes = array();
     $config_prefixes = array();
-    foreach ($xml->xpath('//testsuite/directory') ?: array() as $dir) {
+    foreach ($suite_nodes as $suite) {
+      foreach ($suite->xpath('./directory') ?: array() as $dir) {
         $raw = trim((string) $dir);${directoryRestriction}
         $config_dirs[] = $raw[0] === '/' ? rtrim($raw, '/') : rtrim($base . '/' . $raw, '/');
         foreach (explode(',', (string) ($dir['suffix'] ?? '')) as $suffix) {
@@ -137,18 +149,23 @@ function phpunitConfigDiscoveryPhp(options: PhpunitConfigDiscoveryPhpOptions): s
                 $config_prefixes[] = $prefix;
             }
         }
+      }
     }
-    foreach ($xml->xpath('//testsuite/exclude') ?: array() as $exclude) {
+    foreach ($suite_nodes as $suite) {
+      foreach ($suite->xpath('./exclude') ?: array() as $exclude) {
         $raw = trim((string) $exclude);
         if ($raw !== '') {
             $excludes[] = $raw[0] === '/' ? rtrim($raw, '/') : rtrim($base . '/' . $raw, '/');
         }
+      }
     }
-    foreach ($xml->xpath('//testsuite/file') ?: array() as $file) {
+    foreach ($suite_nodes as $suite) {
+      foreach ($suite->xpath('./file') ?: array() as $file) {
         $raw = trim((string) $file);
         if ($raw !== '') {
             $files[] = $raw[0] === '/' ? $raw : $base . '/' . $raw;
         }
+      }
     }
     if (!empty($config_dirs)) {
         $directories = $config_dirs;
@@ -1227,13 +1244,28 @@ ${phpunitConfigDiscoveryPhp({
 
 ${phpunitDiscoveryPhp("wp_codebox_phpunit_discover", "pg_log")}
 
+function wp_codebox_requested_phpunit_suites($args): array {
+    $suites = array();
+    if (!is_array($args)) {
+        return $suites;
+    }
+    foreach ($args as $index => $arg) {
+        if ($arg === '--testsuite' && isset($args[$index + 1])) {
+            $suites = array_merge($suites, explode(',', (string) $args[$index + 1]));
+        } elseif (is_string($arg) && strpos($arg, '--testsuite=') === 0) {
+            $suites = array_merge($suites, explode(',', substr($arg, strlen('--testsuite='))));
+        }
+    }
+    return array_values(array_filter(array_map('trim', $suites), static function($suite) { return $suite !== ''; }));
+}
+
 pg_stage_begin('discover_tests');
 try {
     $test_dir = $test_root;
     if (!is_dir($test_dir)) {
         throw new RuntimeException('configured PHPUnit test root is not a readable directory: ' . $test_dir);
     }
-    list($directories, $suffixes, $prefixes, $excludes, $configured_files) = wp_codebox_phpunit_parse_config(${JSON.stringify(options.phpunitXml)}, $test_dir);
+    list($directories, $suffixes, $prefixes, $excludes, $configured_files) = wp_codebox_phpunit_parse_config(${JSON.stringify(options.phpunitXml)}, $test_dir, wp_codebox_requested_phpunit_suites($phpunit_args_raw));
     $test_files = wp_codebox_phpunit_discover($directories, $suffixes, $prefixes, $excludes, $configured_files);
     $test_files = pg_filter_changed_test_files($test_files, $changed_test_files_raw, $test_dir);
     if ($selected_test_file !== '') {

packages/runtime-playground/src/wordpress-command-runners.ts

@@ -913,6 +913,7 @@ export async function runPhpunitCommand({
   const bootstrapMode = argValue(args, "bootstrap-mode")?.trim() || "managed"
   const autoloadFile = argValue(args, "autoload-file")?.trim() || (bootstrapMode === "project" ? "" : "/wp-codebox-vendor/autoload.php")
   const autoloadFileRole = argValue(args, "autoload-file-role")?.trim() === "harness" ? "harness" : undefined
+  const phpunitArgs = jsonArrayArg(args, "phpunit-args-json").filter((value): value is string => typeof value === "string")
   const processIdentity = boundedProcessIdentity(spec.processIdentity)
   const resultFile = processIdentity ? `/tmp/wp-codebox-phpunit-result-${processIdentity}.txt` : PLUGIN_PHPUNIT_RESULT_FILE
   const diagnosticHostFile = `/wordpress/wp-content/plugins/${pluginSlug}/.pg-test-result${processIdentity ? `-${processIdentity}` : ""}.txt`
@@ -928,7 +929,7 @@ export async function runPhpunitCommand({
     phpunitXmlIsDefault: phpunitXmlArg === undefined || booleanArg(args, "phpunit-xml-default"),
     selectedTestFile: argValue(args, "test-file")?.trim() || "",
     changedTestFiles: changedTestFilesArg(args),
-    phpunitArgs: jsonArrayArg(args, "phpunit-args-json").filter((value): value is string => typeof value === "string"),
+    phpunitArgs,
     env: jsonObjectArg(args, "env-json"),
     wpConfigDefines: jsonObjectArg(args, "wp-config-defines-json"),
     dependencyMounts: commaListArg(args, "dependency-mounts"),
@@ -969,6 +970,9 @@ export async function runPhpunitCommand({
   if (structured) {
     throw attachPlaygroundDiagnostics(new Error("wordpress.phpunit terminated before completing bootstrap"), "wordpress.phpunit structured diagnostics", structured)
   }
+  if (!phpunitArgs.includes("--list-tests") && !/\bOK(?:, but there were issues!)? \([1-9]\d* tests?,/m.test(response.text)) {
+    throw new Error("wordpress.phpunit exited successfully without a non-zero PHPUnit test summary")
+  }
 
   return response.text
 }

tests/phpunit-project-autoload.test.ts

@@ -136,6 +136,27 @@ echo "ok\n";
   assert.equal(execFileSync("php", [scriptPath], { encoding: "utf8" }), "ok\n")
 }
 
+function assertNamedTestsuiteScopesDiscovery(source: string): void {
+  const tempDir = mkdtempSync(join(tmpdir(), "wp-codebox-named-testsuite-"))
+  const selected = join(tempDir, "SelectedTest.php")
+  const unrelated = join(tempDir, "UnrelatedTest.php")
+  const config = join(tempDir, "phpunit.xml")
+  const scriptPath = join(tempDir, "assert-named-testsuite.php")
+  writeFileSync(selected, "<?php // selected\n")
+  writeFileSync(unrelated, "<?php // unrelated\n")
+  writeFileSync(config, `<phpunit><testsuites><testsuite name="selected"><file>SelectedTest.php</file></testsuite><testsuite name="unrelated"><file>UnrelatedTest.php</file></testsuite></testsuites></phpunit>`)
+  writeFileSync(scriptPath, `<?php
+function pg_log($message) {}
+${extractPhpFunction(source, "wp_codebox_phpunit_parse_config")}
+list($directories, $suffixes, $prefixes, $excludes, $files) = wp_codebox_phpunit_parse_config(${phpString(config)}, ${phpString(tempDir)}, array('selected'));
+if ($directories !== array() || $files !== array(${phpString(selected)})) {
+    throw new RuntimeException('named testsuite discovery escaped its configured scope: ' . json_encode(array($directories, $files)));
+}
+echo "ok\n";
+`)
+  assert.equal(execFileSync("php", [scriptPath], { encoding: "utf8" }), "ok\n")
+}
+
 function assertChangedScopeNoOp(source: string, filterFunctionName: string, relativeFunctionName: string): void {
   const tempDir = mkdtempSync(join(tmpdir(), "wp-codebox-changed-scope-"))
   const scriptPath = join(tempDir, "assert-changed-scope.php")
@@ -484,14 +505,15 @@ assert.ok(projectModeCode.includes("$bootstrap_real = pg_project_bootstrap_real_
 assert.ok(projectModeCode.includes("function pg_project_bootstrap_from_config(string &$xml_path, bool $xml_is_default): string"))
 assertProjectBootstrapConfigResolution(projectModeCode, implicitProjectConfigCode)
 assert.ok(projectModeCode.includes("NOTICE:project bootstrap not declared; continuing without one"), "project mode permits PHPUnit configurations that do not declare a bootstrap")
-assert.ok(projectModeCode.includes("foreach ($xml->xpath('//testsuite/file') ?: array() as $file)"))
+assert.ok(projectModeCode.includes("foreach ($suite->xpath('./file') ?: array() as $file)"))
 assert.ok(projectModeCode.includes("list($directories, $suffixes, $prefixes, $excludes, $configured_files) = wp_codebox_phpunit_parse_config"))
 assert.ok(projectModeCode.includes("$test_files = wp_codebox_phpunit_discover($directories, $suffixes, $prefixes, $excludes, $configured_files);"))
 assert.ok(projectModeCode.includes("' files=' . count($configured_files)"))
 assert.equal(projectModeCode.match(/return array\(\$directories, \$suffixes, \$prefixes, \$excludes\);/g)?.length ?? 0, 0)
 assert.equal(projectModeCode.match(/return \$return_values\(\);/g)?.length, 1)
 assert.match(projectModeCode, /configured PHPUnit test root is not a readable directory/)
 assertPhpunitConfigurationAndDiscoveryFailures(projectModeCode, "wp_codebox_phpunit_parse_config", "wp_codebox_phpunit_discover", "pg_log", false)
+assertNamedTestsuiteScopesDiscovery(projectModeCode)
 assertChangedScopeNoOp(projectModeCode, "pg_filter_changed_test_files", "pg_component_relative_path")
 assertSelectedTestFileResolution(projectModeCode)
 
@@ -546,7 +568,7 @@ await runPhpunitCommand({
   mounts: [],
   runPlaygroundCommand: async (_command, _server, input) => {
     capturedCanonicalHarnessCode = input.code
-    return { text: "ok", exitCode: 0 }
+    return { text: "OK (1 test, 1 assertion)", exitCode: 0 }
   },
   runtimeSpec: phpunitRuntimeSpec,
   server: { playground: {} } as never,
@@ -574,7 +596,7 @@ await runPhpunitCommand({
   mounts: [],
   runPlaygroundCommand: async (_command, _server, input) => {
     capturedExplicitCode = input.code
-    return { text: "ok", exitCode: 0 }
+    return { text: "OK (1 test, 1 assertion)", exitCode: 0 }
   },
   runtimeSpec: phpunitRuntimeSpec,
   server: { playground: {} } as never,

tests/phpunit-runtime-failure-diagnostics.test.ts

@@ -49,4 +49,17 @@ const preBootstrapRecorder = decodedBootstrap.indexOf("STAGE_FATAL:bootstrap:")
 const wordpressBootstrap = decodedBootstrap.indexOf("require_once '/wordpress/wp-load.php';")
 assert.ok(preBootstrapRecorder >= 0 && preBootstrapRecorder < wordpressBootstrap, "fatal diagnostics must be recorded before the WordPress bootstrap boundary")
 
+const emptySuccessArtifactRoot = await mkdtemp(join(tmpdir(), "wp-codebox-phpunit-empty-success-"))
+await assert.rejects(
+  () => runPhpunitCommand({
+    artifactRoot: emptySuccessArtifactRoot,
+    mounts: [],
+    runPlaygroundCommand: async () => ({ exitCode: 0, errors: "", text: "WPCOM Codebox PHPUnit shutdown: mysql_port=unset" }),
+    runtimeSpec: { environment: { kind: "wordpress", name: "test", version: "latest" }, policy: { commands: ["wordpress.phpunit"] } } as never,
+    server: { playground: { readFileAsText: async () => { throw new Error("missing") } } } as never,
+    spec: { command: "wordpress.phpunit", args: ["plugin-slug=demo-plugin"] },
+  }),
+  /exited successfully without a non-zero PHPUnit test summary/,
+)
+
 console.log("phpunit runtime failure diagnostics ok")

tests/runtime-services.test.ts

@@ -18,6 +18,9 @@ assert.throws(() => parseLoopbackPort("0.0.0.0:3306"), /loopback/)
 
 const valid = validateWorkspaceRecipeJsonSchema({ schema: "wp-codebox/workspace-recipe/v1", inputs: { services: [service] }, workflow: { steps: [{ command: "wordpress.run-php" }] } })
 assert.equal(valid.valid, true)
+const aliasedPortService = { ...service, outputs: { ...service.outputs, port: ["DB_PORT", "TC_MYSQL_PORT"] } }
+assert.equal(validateWorkspaceRecipeJsonSchema({ schema: "wp-codebox/workspace-recipe/v1", inputs: { services: [aliasedPortService] }, workflow: { steps: [{ command: "wordpress.run-php" }] } }).valid, true)
+assert.equal(validateWorkspaceRecipeJsonSchema({ schema: "wp-codebox/workspace-recipe/v1", inputs: { services: [{ ...service, outputs: { port: [] } }] }, workflow: { steps: [{ command: "wordpress.run-php" }] } }).valid, false)
 const unsafe = validateWorkspaceRecipeJsonSchema({ schema: "wp-codebox/workspace-recipe/v1", inputs: { services: [{ ...service, outputs: { port: "bad-name" } }] }, workflow: { steps: [{ command: "wordpress.run-php" }] } })
 assert.equal(unsafe.valid, false)
 const emptyRootService = { ...service, configuration: { rootAuthentication: "empty-password" as const } }
@@ -103,6 +106,10 @@ const dependencies: RuntimeServiceDependencies = {
 const provisioned = await provisionRuntimeServices([service], { dependencies })
 assert.equal(provisioned.env.DB_PORT, "41001")
 assert.equal(provisioned.env.DB_PASSWORD, Buffer.alloc(24, 7).toString("base64url"))
+const aliasedPort = await provisionRuntimeServices([aliasedPortService], { dependencies })
+assert.equal(aliasedPort.env.DB_PORT, "41001")
+assert.equal(aliasedPort.env.TC_MYSQL_PORT, "41001")
+await aliasedPort.release()
 const runCall = calls.find((call) => call.args[0] === "run")
 assert.ok(runCall?.args.includes("MYSQL_PASSWORD"))
 assert.ok(runCall?.args.includes("127.0.0.1::3306"), "Docker publishes MySQL on a loopback ephemeral port")
@@ -118,7 +125,7 @@ assert.equal(runCall?.env?.DOCKER_HOST, process.env.DOCKER_HOST, "Docker provide
 assert.equal(calls[0]?.args[0], "image", "the provider checks the image before starting the service")
 await provisioned.release()
 await provisioned.release()
-assert.equal(calls.filter((call) => call.args[0] === "rm").length, 1, "release is idempotent")
+assert.equal(calls.filter((call) => call.args[0] === "rm").length, 2, "each service is released exactly once")
 
 const emptyRootCalls: Array<{ args: string[]; env?: NodeJS.ProcessEnv }> = []
 const emptyRootDependencies: RuntimeServiceDependencies = {