Capture malformed PHPUnit bootstrap payloads (#1933)

* Capture malformed PHPUnit bootstrap payloads

* Treat empty PHPUnit suites as successful

Author: chubes4

packages/runtime-playground/src/php-bootstrap.ts

@@ -27,7 +27,7 @@ export function bootstrapPhpCode(spec: RuntimeCreateSpec, code: string, args: st
 
   const command = splitLeadingStrictTypesDeclare(code)
 
-  return `<?php
+  const bootstrapped = `<?php
 ${command.strictTypesDeclare ? `${command.strictTypesDeclare}\n` : ""}${phpFatalDiagnosticPhp()}
 ${failureDiagnosticFile ? phpFailureDiagnosticFilePhp(failureDiagnosticFile) : ""}
 ${phpCliStreamConstants()}
@@ -43,6 +43,18 @@ ${wpCliBridge ? `putenv(${JSON.stringify(`WP_CODEBOX_TERMINAL_ACTION_URL=${wpCli
 putenv(${JSON.stringify(`WP_CODEBOX_TERMINAL_ACTION_TOKEN=${wpCliBridge.token}`)});
 ` : ""}
 ${command.body}`
+
+  return failureDiagnosticFile ? phpFailureDiagnosticWrapperPhp(bootstrapped, failureDiagnosticFile) : bootstrapped
+}
+
+function phpFailureDiagnosticWrapperPhp(code: string, path: string): string {
+  return `<?php
+try {
+    eval('?>' . base64_decode(${JSON.stringify(Buffer.from(code, "utf8").toString("base64"))}));
+} catch (Throwable $wp_codebox_bootstrap_throwable) {
+    @file_put_contents(${JSON.stringify(path)}, 'STAGE_FAIL:bootstrap:' . get_class($wp_codebox_bootstrap_throwable) . ': ' . $wp_codebox_bootstrap_throwable->getMessage() . ' at ' . $wp_codebox_bootstrap_throwable->getFile() . ':' . $wp_codebox_bootstrap_throwable->getLine() . "\\n", FILE_APPEND);
+    throw $wp_codebox_bootstrap_throwable;
+}`
 }
 
 function phpFailureDiagnosticFilePhp(path: string): string {

packages/runtime-playground/src/phpunit-command-handlers.ts

@@ -1236,7 +1236,7 @@ try {
         pg_log('NO_TEST_FILES');
         pg_log('NOTICE:tests directory not found at ' . $test_dir);
         pg_stage_ok('discover_tests');
-        exit(1);
+        exit(0);
     }
     list($directories, $suffixes, $prefixes, $excludes, $configured_files) = wp_codebox_phpunit_parse_config(${JSON.stringify(options.phpunitXml)}, $test_dir);
     $test_files = wp_codebox_phpunit_discover($directories, $suffixes, $prefixes, $excludes, $configured_files);
@@ -1261,7 +1261,7 @@ try {
     if (empty($test_files)) {
         pg_log('NO_TEST_FILES');
         pg_stage_ok('discover_tests');
-        exit(1);
+        exit(0);
     }
     pg_stage_ok('discover_tests');
 } catch (Throwable $e) {
@@ -1587,7 +1587,7 @@ try {
     if (empty($test_files)) {
         core_pg_log('NO_TEST_FILES');
         core_pg_stage_ok('discover_tests');
-        exit(1);
+        exit(0);
     }
     core_pg_stage_ok('discover_tests');
 } catch (Throwable $e) {

tests/phpunit-project-autoload.test.ts

@@ -358,6 +358,7 @@ assert.ok(projectModeCode.includes("$test_files = wp_codebox_phpunit_discover($d
 assert.ok(projectModeCode.includes("' files=' . count($configured_files)"))
 assert.equal(projectModeCode.match(/return array\(\$directories, \$suffixes, \$prefixes, \$excludes\);/g)?.length ?? 0, 0)
 assert.equal(projectModeCode.match(/return \$return_values\(\);/g)?.length, 3)
+assert.match(projectModeCode, /if \(empty\(\$test_files\)\) \{\s+pg_log\('NO_TEST_FILES'\);\s+pg_stage_ok\('discover_tests'\);\s+exit\(0\);/)
 assertPhpunitParseConfigFallbacksReturnFiveTuple(projectModeCode, "wp_codebox_phpunit_parse_config", "pg_log")
 assertSelectedTestFileResolution(projectModeCode)
 
@@ -466,6 +467,7 @@ assert.ok(coreModeCode.includes("list($directories, $suffixes, $prefixes, $exclu
 assert.ok(coreModeCode.includes("$test_files = core_pg_discover_tests($directories, $suffixes, $prefixes, $excludes, $configured_files);"))
 assert.equal(coreModeCode.match(/return array\(\$directories, \$suffixes, \$prefixes, \$excludes\);/g)?.length ?? 0, 0)
 assert.equal(coreModeCode.match(/return \$return_values\(\);/g)?.length, 3)
+assert.match(coreModeCode, /if \(empty\(\$test_files\)\) \{\s+core_pg_log\('NO_TEST_FILES'\);\s+core_pg_stage_ok\('discover_tests'\);\s+exit\(0\);/)
 assertPhpunitParseConfigFallbacksReturnFiveTuple(coreModeCode, "core_pg_parse_phpunit_config", "core_pg_log")
 for (const privateConstructor of [true, false]) {
   assertDiscoveredTestExecutes(projectModeCode, "pg", privateConstructor)

tests/phpunit-runtime-failure-diagnostics.test.ts

@@ -42,8 +42,11 @@ const captured = await readFile(join(artifactRoot, "files", "phpunit", ".pg-test
 assert.match(captured, /Bootstrap failed with token: \[redacted\]/)
 assert.doesNotMatch(captured, new RegExp(secret))
 
-const preBootstrapRecorder = submittedCode.indexOf("STAGE_FATAL:bootstrap:")
-const wordpressBootstrap = submittedCode.indexOf("require_once '/wordpress/wp-load.php';")
+const encodedBootstrap = submittedCode.match(/base64_decode\("([A-Za-z0-9+/=]+)"\)/)?.[1]
+assert.ok(encodedBootstrap, "PHPUnit payload must execute inside the bootstrap diagnostic wrapper")
+const decodedBootstrap = Buffer.from(encodedBootstrap, "base64").toString("utf8")
+const preBootstrapRecorder = decodedBootstrap.indexOf("STAGE_FATAL:bootstrap:")
+const wordpressBootstrap = decodedBootstrap.indexOf("require_once '/wordpress/wp-load.php';")
 assert.ok(preBootstrapRecorder >= 0 && preBootstrapRecorder < wordpressBootstrap, "fatal diagnostics must be recorded before the WordPress bootstrap boundary")
 
 console.log("phpunit runtime failure diagnostics ok")

tests/playground-phpunit-bootstrap-failure.integration.test.ts

@@ -12,6 +12,7 @@ const fatalMuPlugin = join(root, "fatal-bootstrap.php")
 const recipePath = join(root, "recipe.json")
 const artifactsPath = join(root, "artifacts")
 const exitArtifactsPath = join(root, "exit-artifacts")
+const parseArtifactsPath = join(root, "parse-artifacts")
 
 try {
   await writeFile(fatalMuPlugin, `<?php\ntrigger_error('PHPUnit bootstrap fixture token: ${secret}', E_USER_ERROR);\n`)
@@ -65,6 +66,26 @@ try {
   const exitDiagnostic = await readFile(join(exitArtifactsPath, exitRuntimeDirectory, "files", "phpunit", ".pg-test-result.txt"), "utf8")
   assert.match(exitDiagnostic, /STAGE_DIE:bootstrap:PHPUnit bootstrap exit fixture token: \[redacted\]/)
   assert.doesNotMatch(exitDiagnostic, new RegExp(secret))
+
+  await writeFile(recipePath, `${JSON.stringify({
+    schema: "wp-codebox/workspace-recipe/v1",
+    runtime: { backend: "wordpress-playground", wp: "6.5", blueprint: { steps: [] } },
+    inputs: { mounts: [] },
+    workflow: {
+      steps: [{ command: "wordpress.phpunit", args: ["plugin-slug=bootstrap-failure-fixture", "code=<?php function malformed("] }],
+    },
+  })}\n`)
+  const parseOutput = await runFailedRecipe(parseArtifactsPath)
+  assert.equal(parseOutput.success, false, JSON.stringify(parseOutput))
+  const parseMessage = parseOutput.error?.message ?? ""
+  assert.match(parseMessage, /wordpress\.phpunit structured diagnostics/)
+  assert.match(parseMessage, /ParseError/)
+
+  const parseLatest = JSON.parse(await readFile(join(parseArtifactsPath, "latest-runtime.json"), "utf8")) as { paths?: { runtimeDirectory?: string } }
+  const parseRuntimeDirectory = parseLatest.paths?.runtimeDirectory
+  assert.ok(parseRuntimeDirectory, "malformed recipe must retain a runtime artifact directory")
+  const parseDiagnostic = await readFile(join(parseArtifactsPath, parseRuntimeDirectory, "files", "phpunit", ".pg-test-result.txt"), "utf8")
+  assert.match(parseDiagnostic, /STAGE_FAIL:bootstrap:ParseError/)
 } finally {
   await rm(root, { recursive: true, force: true })
 }