Tighten benchmark safety planning

Resolve public-looking benchmark targets through DNS before applying the
safety gate, and keep unresolved or mixed-public answers in the public tier.
Require unsafe public-target overrides to name the target on the command line
and to set load and duration explicitly.

Make dry-run mode perform discovery planning for runnable scenarios while
still avoiding benchmark load, so inbox plans show the resolved actor and
inbox destination before a real run.

#744
#784

Assisted-by: Codex:gpt-5.5

Author: dahlia

packages/cli/src/bench/action.test.ts

@@ -19,6 +19,7 @@ async function spawnTarget() {
     benchmarkMode: true,
   });
   let keyPairs: CryptoKeyPair[] | undefined;
+  const requests: { method: string; path: string }[] = [];
   federation
     .setActorDispatcher("/users/{identifier}", async (ctx, identifier) => {
       if (identifier !== "alice") return null;
@@ -51,6 +52,8 @@ async function spawnTarget() {
     hostname: "127.0.0.1",
     silent: true,
     fetch: (request: Request) => {
+      const url = new URL(request.url);
+      requests.push({ method: request.method, path: url.pathname });
       if (request.method === "POST") {
         inboxUserAgent = request.headers.get("user-agent");
       }
@@ -61,6 +64,7 @@ async function spawnTarget() {
   return {
     url: new URL(server.url!),
     inboxUserAgent: () => inboxUserAgent,
+    requests: () => requests.slice(),
     close: () => server.close(true),
   };
 }
@@ -213,12 +217,48 @@ test("runBench - dry run prints a plan and sends nothing", async () => {
     });
     assert.strictEqual(code, 0);
     assert.match(output, /dry run/i);
-    assert.match(output, /No requests were sent/);
+    assert.match(output, /\/inbox/);
+    assert.match(output, /No benchmark load was sent/);
+    const requests = target.requests();
+    assert.ok(requests.some((r) => r.method === "GET"));
+    assert.ok(!requests.some((r) => r.method === "POST"));
   } finally {
     await target.close();
   }
 });
 
+test("runBench - unsafe override requires an explicit CLI target", async () => {
+  const file = await writeSuite(`version: 1
+target: https://example.com
+scenarios:
+  - name: wf
+    type: webfinger
+    recipient: "acct:alice@example.com"
+    load: { rate: 1/s }
+    duration: 1ms
+`);
+  let code = -1;
+  let message = "";
+  await runBench(command({ scenario: file, allowUnsafeTarget: true }), {
+    exit: (c) => {
+      code = c;
+    },
+    writeOutput: () => Promise.resolve(),
+    log: (m) => {
+      message = m;
+    },
+    fetch: (input) => {
+      const url = new URL(input instanceof Request ? input.url : input);
+      if (url.pathname.includes("/bench/stats")) {
+        return Promise.resolve(new Response("not found", { status: 404 }));
+      }
+      return Promise.resolve(new Response("ok"));
+    },
+  });
+  assert.strictEqual(code, 2);
+  assert.match(message, /--target/);
+});
+
 test("runBench - refuses an unsafe public target (exit 2)", async () => {
   const file = await writeSuite(`version: 1
 target: https://example.com

packages/cli/src/bench/action.ts

@@ -1,8 +1,10 @@
 import { writeFile } from "node:fs/promises";
+import type { DocumentLoader } from "@fedify/vocab-runtime";
 import process from "node:process";
 import { getContextLoader, getDocumentLoader } from "../docloader.ts";
 import { buildFleet } from "./actor/fleet.ts";
 import type { BenchCommand } from "./command.ts";
+import { discoverInbox, selectInbox } from "./discovery/discover.ts";
 import {
   buildReport,
   buildScenarioResult,
@@ -18,20 +20,25 @@ import {
   type ResolvedScenario,
   type ResolvedSuite,
 } from "./scenario/normalize.ts";
-import type { Suite } from "./scenario/types.ts";
+import type { LoadConfig, Suite } from "./scenario/types.ts";
 import { validateSuite } from "./scenario/validate.ts";
 import {
   assertInboxDestinationAllowed,
   assertTargetAllowed,
+  assertUnsafeOverrideAllowed,
   UnsafeTargetError,
 } from "./safety/gate.ts";
-import { classifyTarget } from "./safety/tiers.ts";
+import {
+  classifyResolvedTarget,
+  type ResolveTargetAddresses,
+} from "./safety/tiers.ts";
 import { runnerFor } from "./scenarios/registry.ts";
 import {
   resolveAdvertiseHost,
   spawnSyntheticServer,
   type SyntheticServer,
 } from "./server/synthetic.ts";
+import { convertUrlIfHandle } from "../webfinger/lib.ts";
 
 /** Injectable dependencies for {@link runBench}, overridable in tests. */
 export interface RunBenchDeps {
@@ -46,6 +53,8 @@ export interface RunBenchDeps {
   readonly log?: (message: string) => void;
   /** Fetch implementation. */
   readonly fetch?: typeof fetch;
+  /** Hostname resolver used for target risk classification. */
+  readonly resolveTargetAddresses?: ResolveTargetAddresses;
 }
 
 /** The scenario types that need the synthetic actor/key server. */
@@ -109,19 +118,26 @@ export default async function runBench(
     return void exit(2);
   }
 
-  if (command.dryRun) {
-    await writeOutput(renderPlan(suite), command.output);
-    return void exit(0);
-  }
-
-  const tier = classifyTarget(suite.target);
+  const tier = await classifyResolvedTarget(
+    suite.target,
+    deps.resolveTargetAddresses,
+  );
   const probe = await probeBenchmarkMode(suite.target, fetchImpl);
   try {
+    if (!command.dryRun) {
+      assertUnsafeOverrideAllowed({
+        tier,
+        benchmarkMode: probe.benchmarkMode,
+        allowUnsafe: command.allowUnsafeTarget,
+        explicitCliTarget: command.target != null,
+        scenarios: unsafeOverrideScenarios(validated),
+      });
+    }
     assertTargetAllowed({
       tier,
       benchmarkMode: probe.benchmarkMode,
       allowUnsafe: command.allowUnsafeTarget,
-      dryRun: false,
+      dryRun: command.dryRun,
     });
   } catch (error) {
     if (error instanceof UnsafeTargetError) {
@@ -136,20 +152,6 @@ export default async function runBench(
   // same-machine (loopback) target; a non-loopback target needs an advertised,
   // reachable host (--advertise-host).  Without one, refuse signed scenarios
   // rather than let every signed delivery fail key lookup.
-  if (
-    tier !== "loopback" && command.advertiseHost == null &&
-    suite.scenarios.some((s) => SIGNED_TYPES.has(s.type))
-  ) {
-    log(
-      "Signed scenarios (inbox) need the benchmark's synthetic actor server to " +
-        "be reachable from the target.  A loopback target reaches it " +
-        "automatically; for a non-loopback target, pass --advertise-host with " +
-        "an address the target can reach (the synthetic server then binds all " +
-        "interfaces), or use a read scenario such as webfinger.",
-    );
-    return void exit(2);
-  }
-
   const allowPrivateAddress = tier !== "public";
   const documentLoader = await getDocumentLoader({
     allowPrivateAddress,
@@ -170,6 +172,43 @@ export default async function runBench(
       advertised: command.advertiseHost != null,
     });
 
+  if (command.dryRun) {
+    try {
+      await writeOutput(
+        await renderPlan(suite, {
+          documentLoader,
+          contextLoader,
+          allowPrivateAddress,
+          assertDestinationAllowed,
+        }),
+        command.output,
+      );
+      return void exit(0);
+    } catch (error) {
+      log(error instanceof Error ? error.message : String(error));
+      return void exit(2);
+    }
+  }
+
+  // The target dereferences the synthetic actor server while verifying
+  // signatures.  By default that server is loopback-only, reachable just by a
+  // same-machine (loopback) target; a non-loopback target needs an advertised,
+  // reachable host (--advertise-host).  Without one, refuse signed scenarios
+  // rather than let every signed delivery fail key lookup.
+  if (
+    tier !== "loopback" && command.advertiseHost == null &&
+    suite.scenarios.some((s) => SIGNED_TYPES.has(s.type))
+  ) {
+    log(
+      "Signed scenarios (inbox) need the benchmark's synthetic actor server to " +
+        "be reachable from the target.  A loopback target reaches it " +
+        "automatically; for a non-loopback target, pass --advertise-host with " +
+        "an address the target can reach (the synthetic server then binds all " +
+        "interfaces), or use a read scenario such as webfinger.",
+    );
+    return void exit(2);
+  }
+
   let fleet: SyntheticServer | undefined;
   const startedAt = new Date().toISOString();
   try {
@@ -278,7 +317,17 @@ async function defaultWriteOutput(
   await writeFile(outputPath, content, { encoding: "utf-8" });
 }
 
-function renderPlan(suite: ResolvedSuite): string {
+interface DryRunPlanContext {
+  readonly documentLoader: DocumentLoader;
+  readonly contextLoader: DocumentLoader;
+  readonly allowPrivateAddress: boolean;
+  readonly assertDestinationAllowed: (url: URL) => void;
+}
+
+async function renderPlan(
+  suite: ResolvedSuite,
+  context: DryRunPlanContext,
+): Promise<string> {
   const lines = [
     "Fedify benchmark plan (dry run)",
     "",
@@ -289,8 +338,13 @@ function renderPlan(suite: ResolvedSuite): string {
     lines.push(
       `- ${scenario.name} (${scenario.type}): ${describePlan(scenario)}`,
     );
+    lines.push(...await describeDiscoveryPlan(scenario, suite, context));
   }
-  lines.push("", "No requests were sent.");
+  lines.push(
+    "",
+    "No benchmark load was sent.  Discovery and stats probe requests may " +
+      "have been sent.",
+  );
   return `${lines.join("\n")}\n`;
 }
 
@@ -300,3 +354,89 @@ function describePlan(scenario: ResolvedScenario): string {
     : `closed-loop concurrency ${scenario.load.concurrency}`;
   return `${load}, duration ${scenario.durationMs}ms, signing ${scenario.signing}`;
 }
+
+async function describeDiscoveryPlan(
+  scenario: ResolvedScenario,
+  suite: ResolvedSuite,
+  context: DryRunPlanContext,
+): Promise<string[]> {
+  switch (scenario.type) {
+    case "inbox":
+      return await describeInboxDiscoveryPlan(scenario, context);
+    case "webfinger":
+      return describeWebFingerPlan(scenario, suite.target);
+    default:
+      return ["  discovery: not available for this scenario type"];
+  }
+}
+
+async function describeInboxDiscoveryPlan(
+  scenario: ResolvedScenario,
+  context: DryRunPlanContext,
+): Promise<string[]> {
+  const lines: string[] = [];
+  for (const recipient of scenario.recipients) {
+    const discovered = await discoverInbox(recipient, {
+      documentLoader: context.documentLoader,
+      contextLoader: context.contextLoader,
+      allowPrivateAddress: context.allowPrivateAddress,
+    });
+    const inbox = selectInbox(discovered, scenario.inbox);
+    lines.push(
+      `  recipient ${recipient}: actor ${discovered.actorUri.href}, ` +
+        `inbox ${inbox.href}`,
+    );
+    lines.push(
+      `  destination safety: ${describeDestinationSafety(inbox, context)}`,
+    );
+  }
+  return lines;
+}
+
+function describeWebFingerPlan(
+  scenario: ResolvedScenario,
+  target: URL,
+): string[] {
+  const recipients = scenario.recipients.length > 0
+    ? scenario.recipients
+    : [target.href];
+  return recipients.map((recipient) => {
+    const resource = convertUrlIfHandle(recipient).href;
+    const url = new URL("/.well-known/webfinger", target);
+    url.searchParams.set("resource", resource);
+    return `  webfinger ${resource}: GET ${url.href}`;
+  });
+}
+
+function describeDestinationSafety(
+  inbox: URL,
+  context: DryRunPlanContext,
+): string {
+  try {
+    context.assertDestinationAllowed(inbox);
+    return "allowed";
+  } catch (error) {
+    if (error instanceof UnsafeTargetError) {
+      return `would be refused: ${error.message}`;
+    }
+    throw error;
+  }
+}
+
+function unsafeOverrideScenarios(
+  suite: Suite,
+): Parameters<typeof assertUnsafeOverrideAllowed>[0]["scenarios"] {
+  const defaultDuration = suite.defaults?.duration != null;
+  const defaultLoad = hasExplicitLoad(suite.defaults?.load);
+  return suite.scenarios.map((scenario) => ({
+    name: scenario.name,
+    explicitDuration: scenario.duration != null || defaultDuration,
+    explicitLoad: hasExplicitLoad(scenario.load) || defaultLoad,
+  }));
+}
+
+function hasExplicitLoad(load: LoadConfig | undefined): boolean {
+  return load != null &&
+    (("rate" in load && load.rate != null) ||
+      ("concurrency" in load && load.concurrency != null));
+}