Merge pull request #2026 from Automattic/feat/2019-adversarial-recipe-orchestration

Run adversarial campaigns from workspace recipes

Author: chubes4

docs/recipe-contract.md

@@ -86,6 +86,7 @@ top-level fields:
 - `inputs`
 - `workflow`
 - `fuzzRun`
+- `adversarialCampaigns`
 - `artifacts`
 - `probes`
 - `metadata`
@@ -124,6 +125,37 @@ Use `inputs.workspace_preloads` for generic `agent-runtime/workspace-preload`
 artifact contracts. WP Codebox materializes declared repositories as sandbox
 workspace mounts; callers own the policy that decides which artifacts to pass.
 
+## Adversarial Campaigns
+
+`adversarialCampaigns` additively declares deterministic corpus campaigns without
+changing authored `fuzzRun.cases`. Each corpus action selects a registered
+`caseTemplates` id. Generated plans interpolate `{{case.id}}`, `{{case.input}}`,
+`{{action.input}}`, `{{action.inputBase64}}`, `{{matrix}}`, and
+`{{matrix.<name>}}` into template arguments, then execute the resulting phases
+through the existing runtime-backed fuzz suite, checkpoint, episode, command
+policy, and artifact lifecycle.
+
+The built-in mutator registry contains `scalar`, `structured`, `binary`, and
+`sequence`; the initial neutral oracle registry contains `runtime-status`.
+Required capabilities fail recipe validation before services or runtimes start.
+Optional capabilities are reported with explicit availability in run evidence.
+Concurrency above one requires an `isolated-workers` capability and therefore
+fails closed until the selected recipe runtime provides isolated workers.
+
+Run and replay through stable CLI routes:
+
+```bash
+wp-codebox adversarial run --recipe examples/recipes/adversarial-stateful.json --json
+wp-codebox adversarial replay --recipe recipe.json --replay files/adversarial/<campaign>/replay/<fingerprint>.json --json
+```
+
+Every campaign writes a bounded nested bundle under
+`files/adversarial/<campaign>/`, including the result, fingerprint-addressed
+findings, minimized replay envelopes, a manifest, and secret-scan evidence. The
+parent artifact manifest retains all nested files. Interruption returns bounded
+partial campaign evidence before the recipe runner performs its normal runtime
+and service teardown.
+
 ## External Service Boundaries
 
 Use `inputs.externalServices` to declare reviewer-safe boundaries for services a

examples/recipes/adversarial-stateful.json

@@ -0,0 +1,100 @@
+{
+  "schema": "wp-codebox/workspace-recipe/v1",
+  "runtime": {
+    "backend": "wordpress"
+  },
+  "workflow": {
+    "steps": [
+      {
+        "command": "wordpress.run-php",
+        "args": ["code=delete_option( 'wp_codebox_adversarial_state' );"]
+      }
+    ]
+  },
+  "adversarialCampaigns": [
+    {
+      "schema": "wp-codebox/adversarial-recipe-campaign/v1",
+      "id": "neutral-stateful",
+      "seed": "neutral-stateful-v1",
+      "corpus": [
+        {
+          "id": "option-seed",
+          "actions": [
+            {
+              "type": "option-roundtrip",
+              "input": {
+                "value": "alpha"
+              }
+            }
+          ],
+          "input": {
+            "source": "recipe"
+          },
+          "signals": ["baseline"]
+        }
+      ],
+      "caseTemplates": [
+        {
+          "id": "option-roundtrip",
+          "phases": {
+            "action": [
+              {
+                "command": "wordpress.run-php",
+                "args": ["code=$value = json_decode( base64_decode( '{{action.inputBase64}}' ), true ); update_option( 'wp_codebox_adversarial_state', $value );"]
+              }
+            ],
+            "assert": [
+              {
+                "command": "wordpress.run-php",
+                "args": ["code=if ( get_option( 'wp_codebox_adversarial_state', null ) === null ) { throw new RuntimeException( 'state was not retained' ); }"]
+              }
+            ],
+            "teardown": [
+              {
+                "command": "wordpress.run-php",
+                "args": ["code=delete_option( 'wp_codebox_adversarial_state' );"]
+              }
+            ]
+          }
+        }
+      ],
+      "mutators": ["scalar", "structured", "sequence"],
+      "oracles": [
+        {
+          "id": "runtime-status",
+          "severity": "high"
+        }
+      ],
+      "matrix": [
+        {
+          "name": "storage",
+          "values": ["options"]
+        }
+      ],
+      "concurrency": 1,
+      "budgets": {
+        "maxCases": 8,
+        "maxActionsPerCase": 8,
+        "maxInputBytes": 65536,
+        "maxCaseTimeMs": 10000,
+        "maxWallTimeMs": 120000,
+        "maxArtifactBytes": 10485760
+      },
+      "resetPolicy": {
+        "mode": "checkpoint-per-case",
+        "checkpointName": "neutral-stateful-baseline"
+      },
+      "requiredCapabilities": ["adversarial-campaign", "artifact-export", "command:wordpress.run-php"],
+      "optionalCapabilities": ["transport-faults"],
+      "novelty": {
+        "retainSignals": true
+      },
+      "shrinking": {
+        "enabled": true
+      },
+      "replay": {
+        "selection": "findings"
+      }
+    }
+  ]
+}

package.json

@@ -230,7 +230,7 @@
     "test:recipe-run-provenance": "tsx tests/recipe-run-provenance.test.ts",
     "test:fuzz-run-recipe": "tsx tests/fuzz-run-recipe.test.ts",
     "test:fuzz-suite-runner": "tsx tests/fuzz-suite-runner.test.ts",
-    "test:adversarial-runtime": "tsx --test tests/transport-faults.test.ts tests/adversarial-campaign.test.ts tests/adversarial-browser.test.ts tests/browser-clock-control.test.ts",
+    "test:adversarial-runtime": "tsx --test tests/transport-faults.test.ts tests/adversarial-campaign.test.ts tests/adversarial-browser.test.ts tests/browser-clock-control.test.ts tests/adversarial-recipe-orchestration.test.ts",
     "test:playground-fuzz-suite-public": "tsx tests/playground-fuzz-suite-public.test.ts",
     "test:nested-fuzz-suite-recipe-command": "tsx tests/nested-fuzz-suite-recipe-command.test.ts",
     "test:wordpress-fuzz-suite-builders": "tsx tests/wordpress-fuzz-suite-builders.test.ts",