Add allowClientError to Scenario interface for expected client failures (#123)

* Add allowClientError to Scenario interface for expected client failures

* Apply suggestion from @pcarleton

Co-authored-by: Paul Carleton <paulcarletonjr@gmail.com>

---------

Co-authored-by: Paul Carleton <paulcarletonjr@gmail.com>

Author: mikekistler

src/index.ts

@@ -207,7 +207,8 @@ program
       const { overallFailure } = printClientResults(
         result.checks,
         verbose,
-        result.clientOutput
+        result.clientOutput,
+        result.allowClientError
       );
 
       if (options.expectedFailures) {

src/runner/client.ts

@@ -97,6 +97,7 @@ export async function runConformanceTest(
   checks: ConformanceCheck[];
   clientOutput: ClientExecutionResult;
   resultDir?: string;
+  allowClientError?: boolean;
 }> {
   let resultDir: string | undefined;
 
@@ -164,7 +165,8 @@ export async function runConformanceTest(
     return {
       checks,
       clientOutput,
-      resultDir
+      resultDir,
+      allowClientError: scenario.allowClientError
     };
   } finally {
     await scenario.stop();
@@ -174,7 +176,8 @@ export async function runConformanceTest(
 export function printClientResults(
   checks: ConformanceCheck[],
   verbose: boolean = false,
-  clientOutput?: ClientExecutionResult
+  clientOutput?: ClientExecutionResult,
+  allowClientError: boolean = false
 ): {
   passed: number;
   failed: number;
@@ -195,7 +198,10 @@ export function printClientResults(
     ? clientOutput.exitCode !== 0
     : false;
   const overallFailure =
-    failed > 0 || warnings > 0 || clientTimedOut || clientExitedWithError;
+    failed > 0 ||
+    warnings > 0 ||
+    clientTimedOut ||
+    (clientExitedWithError && !allowClientError);
 
   if (verbose) {
     // Verbose mode: JSON goes to stdout for piping to jq/jless
@@ -215,7 +221,7 @@ export function printClientResults(
     console.error(`\n⚠️  CLIENT TIMED OUT - Test incomplete`);
   }
 
-  if (clientExitedWithError && !clientTimedOut) {
+  if (clientExitedWithError && !clientTimedOut && !allowClientError) {
     console.error(
       `\n⚠️  CLIENT EXITED WITH ERROR (code ${clientOutput?.exitCode}) - Test may be incomplete`
     );

src/scenarios/client/auth/resource-mismatch.ts

@@ -29,6 +29,7 @@ export class ResourceMismatchScenario implements Scenario {
   name = 'auth/resource-mismatch';
   description =
     'Tests that client rejects when PRM resource does not match server URL';
+  allowClientError = true;
 
   private authServer = new ServerLifecycle();
   private server = new ServerLifecycle();

src/scenarios/client/auth/scope-handling.ts

@@ -479,6 +479,7 @@ export class ScopeRetryLimitScenario implements Scenario {
   name = 'auth/scope-retry-limit';
   description =
     'Tests that client implements retry limits to prevent infinite authorization loops on repeated 403 responses';
+  allowClientError = true;
   private authServer = new ServerLifecycle();
   private server = new ServerLifecycle();
   private checks: ConformanceCheck[] = [];

src/types.ts

@@ -36,6 +36,11 @@ export interface ScenarioUrls {
 export interface Scenario {
   name: string;
   description: string;
+  /**
+   * If true, a non-zero client exit code is expected and will not cause the test to fail.
+   * Use this for scenarios where the client is expected to error (e.g., rejecting invalid auth).
+   */
+  allowClientError?: boolean;
   start(): Promise<ScenarioUrls>;
   stop(): Promise<void>;
   getChecks(): ConformanceCheck[];