feat: Add --mode positive/negative to run only positive or negative playbooks

ludovicianul · ludovicianul · commit b6ae652bdc5b · 2026-03-21T22:32:05.000+02:00
diff --git a/src/main/java/dev/dochia/cli/core/args/FilterArguments.java b/src/main/java/dev/dochia/cli/core/args/FilterArguments.java
@@ -69,6 +69,10 @@ public class FilterArguments {
     @Inject
     ProfileLoader profileLoader;
 
+    enum Mode {
+        ALL, NEGATIVE, POSITIVE
+    }
+
     enum FieldType {
         STRING, NUMBER, INTEGER, BOOLEAN
     }
@@ -174,10 +178,42 @@ enum FormatType {
             description = "Path to custom profile configuration file (YAML)")
     private Path customProfileFile;
 
+    @CommandLine.Option(
+            names = "--mode",
+            description = "Playbook selection mode. Available values:%n" +
+                    "  @|bold ALL|@       - Run all playbooks (default).%n" +
+                    "  @|bold NEGATIVE|@  - Run only negative test scenarios (playbooks expecting 4XX responses: " +
+                    "validation, invalid values, authentication, etc). Excludes happy path playbooks.%n" +
+                    "  @|bold POSITIVE|@  - Run only positive test scenarios (playbooks expecting 2XX responses: " +
+                    "happy path, valid examples, default values). Excludes validation error playbooks.",
+            defaultValue = "ALL",
+            showDefaultValue = CommandLine.Help.Visibility.ALWAYS
+    )
+    private Mode mode;
+
 
     private Map<String, List<String>> skipPathPlaybooks = new HashMap<>();
     private Map<String, Map<String, List<String>>> skipPlaybooksForExtensionMap = new HashMap<>();
 
+
+    /**
+     * Returns if DOCHIA should run only the negative playbooks.
+     *
+     * @return true if DOCHIA should run only the negative playbooks, false otherwise
+     */
+    public boolean isOnly4xxPlaybooks() {
+        return mode == Mode.NEGATIVE;
+    }
+
+    /**
+     * Returns if DOCHIA should run only the positive playbooks.
+     *
+     * @return true if DOCHIA should run only the positive playbooks, false otherwise
+     */
+    public boolean isOnly2xxPlaybooks() {
+        return mode == Mode.POSITIVE;
+    }
+    
     /**
      * Gets the list of fields to skip during processing. If the list is not set, an empty list is returned.
      *
diff --git a/src/main/java/dev/dochia/cli/core/playbook/executor/FieldsIteratorExecutor.java b/src/main/java/dev/dochia/cli/core/playbook/executor/FieldsIteratorExecutor.java
@@ -108,6 +108,11 @@ private void executeTestCase(FieldsIteratorExecutorContext context, String fuzze
         testCaseListener.addExpectedResult(context.getLogger(), "Should return [{}]",
                 context.getExpectedResponseCode() != null ? context.getExpectedResponseCode().asString() : "a response that doesn't match" + matchArguments.getMatchString());
 
+        if (!testCaseListener.shouldContinueExecution(context.getLogger(), context.getExpectedResponseCode())) {
+            testCaseListener.skipTest(context.getLogger(), "Test skipped due to response code filtering");
+            return;
+        }
+
         FuzzingResult fuzzingResult = this.getFuzzingResult(context, fuzzedField, strategy);
 
         HttpResponse response = serviceCaller.call(
diff --git a/src/main/java/dev/dochia/cli/core/playbook/executor/HeadersIteratorExecutor.java b/src/main/java/dev/dochia/cli/core/playbook/executor/HeadersIteratorExecutor.java
@@ -70,6 +70,11 @@ public void execute(HeadersIteratorExecutorContext context) {
                             testCaseListener.addExpectedResult(context.getLogger(), "Should return [{}]",
                                     expectedResponseCode != null ? expectedResponseCode.asString() : "a response that doesn't match" + matchArguments.getMatchString());
 
+                            if (!testCaseListener.shouldContinueExecution(context.getLogger(), expectedResponseCode)) {
+                                testCaseListener.skipTest(context.getLogger(), "Test skipped due to response code filtering");
+                                return;
+                            }
+
                             ServiceData serviceData = ServiceData.builder()
                                     .relativePath(context.getPlaybookData().getPath())
                                     .contractPath(context.getPlaybookData().getContractPath())
diff --git a/src/main/java/dev/dochia/cli/core/playbook/executor/SimpleExecutor.java b/src/main/java/dev/dochia/cli/core/playbook/executor/SimpleExecutor.java
@@ -56,6 +56,11 @@ public void execute(SimpleExecutorContext context) {
             testCaseListener.addScenario(context.getLogger(), context.getScenario());
             testCaseListener.addExpectedResult(context.getLogger(), "Should return {}" + context.getExpectedResult(), context.getExpectedSpecificResponseCode());
 
+            if (!testCaseListener.shouldContinueExecution(context.getLogger(), context.getExpectedResponseCode())) {
+                testCaseListener.skipTest(context.getLogger(), "Test skipped due to response code filtering");
+                return;
+            }
+
             HttpResponse response = serviceCaller.call(
                     ServiceData.builder()
                             .relativePath(context.getPath())
diff --git a/src/main/java/dev/dochia/cli/core/playbook/field/DuplicateKeysFieldsPlaybook.java b/src/main/java/dev/dochia/cli/core/playbook/field/DuplicateKeysFieldsPlaybook.java
@@ -121,6 +121,11 @@ private void runDuplicationTestCase(PlaybookData data, String field, String dupl
         testCaseListener.addExpectedResult(logger,
                 "Service should return a [{}] response", ResponseCodeFamilyPredefined.FOURXX.asString());
 
+        if (!testCaseListener.shouldContinueExecution(logger, ResponseCodeFamilyPredefined.FOURXX)) {
+            testCaseListener.skipTest(logger, "Test skipped due to response code filtering");
+            return;
+        }
+
         HttpResponse response = serviceCaller.call(buildServiceData(data, duplicatedPayload));
         testCaseListener.reportResult(logger, data, response, ResponseCodeFamilyPredefined.FOURXX);
     }
diff --git a/src/main/java/dev/dochia/cli/core/playbook/field/InsertWhitespacesInFieldNamesFieldPlaybook.java b/src/main/java/dev/dochia/cli/core/playbook/field/InsertWhitespacesInFieldNamesFieldPlaybook.java
@@ -59,6 +59,11 @@ private void process(PlaybookData data, String field, String randomWhitespace) {
         testCaseListener.addScenario(logger, "Insert random whitespaces in the field name [{}]", field);
         testCaseListener.addExpectedResult(logger, "Should get a [{}] response code", ResponseCodeFamilyPredefined.FOURXX);
 
+        if (!testCaseListener.shouldContinueExecution(logger, ResponseCodeFamilyPredefined.FOURXX)) {
+            testCaseListener.skipTest(logger, "Test skipped due to response code filtering");
+            return;
+        }
+
         String fuzzedJson = JsonUtils.insertCharactersInFieldKey(data.getPayload(), field, randomWhitespace);
 
         HttpResponse response = serviceCaller.call(ServiceData.builder().relativePath(data.getPath()).headers(data.getHeaders())
diff --git a/src/main/java/dev/dochia/cli/core/playbook/field/NewFieldsPlaybook.java b/src/main/java/dev/dochia/cli/core/playbook/field/NewFieldsPlaybook.java
@@ -62,6 +62,11 @@ private void process(PlaybookData data) {
         testCaseListener.addScenario(logger, "Add new field inside the request: name [{}], value [{}]. All other details are similar to a happy flow", NEW_FIELD, NEW_FIELD);
         testCaseListener.addExpectedResult(logger, "Should get a [{}] response code", expectedResultCode.asString());
 
+        if (!testCaseListener.shouldContinueExecution(logger, expectedResultCode)) {
+            testCaseListener.skipTest(logger, "Test skipped due to response code filtering");
+            return;
+        }
+
         HttpResponse response = serviceCaller.call(ServiceData.builder().relativePath(data.getPath()).headers(data.getHeaders())
                 .payload(fuzzedJson).queryParams(data.getQueryParams()).httpMethod(data.getMethod()).contractPath(data.getContractPath())
                 .contentType(data.getFirstRequestContentType()).pathParamsPayload(data.getPathParamsPayload())
diff --git a/src/main/java/dev/dochia/cli/core/playbook/field/RemoveFieldsPlaybook.java b/src/main/java/dev/dochia/cli/core/playbook/field/RemoveFieldsPlaybook.java
@@ -3,6 +3,7 @@
 import dev.dochia.cli.core.args.FilterArguments;
 import dev.dochia.cli.core.args.ProcessingArguments;
 import dev.dochia.cli.core.http.HttpMethod;
+import dev.dochia.cli.core.http.ResponseCodeFamily;
 import dev.dochia.cli.core.http.ResponseCodeFamilyPredefined;
 import dev.dochia.cli.core.io.ServiceCaller;
 import dev.dochia.cli.core.io.ServiceData;
@@ -89,8 +90,12 @@ private void process(PlaybookData data, List<String> required, Set<String> subse
             testCaseListener.addScenario(logger, "Remove the following fields from request: {}", subset);
 
             boolean hasRequiredFieldsRemove = this.hasRequiredFieldsRemove(required, subset);
+            ResponseCodeFamily expectedResponseCode = ResponseCodeFamilyPredefined.getResultCodeBasedOnRequiredFieldsRemoved(hasRequiredFieldsRemove);
             testCaseListener.addExpectedResult(logger, "Should return [{}] response code as required fields [{}] removed", ResponseCodeFamilyPredefined.getExpectedWordingBasedOnRequiredFields(hasRequiredFieldsRemove));
-
+            if (!testCaseListener.shouldContinueExecution(logger, expectedResponseCode)) {
+                testCaseListener.skipTest(logger, "Test skipped due to response code filtering");
+                return;
+            }
             HttpResponse response = serviceCaller.call(ServiceData.builder().relativePath(data.getPath()).headers(data.getHeaders())
                     .payload(finalJsonPayload).queryParams(data.getQueryParams()).httpMethod(data.getMethod()).contractPath(data.getContractPath())
                     .contentType(data.getFirstRequestContentType()).pathParamsPayload(data.getPathParamsPayload()).build());
diff --git a/src/main/java/dev/dochia/cli/core/playbook/field/ZeroWidthCharsInNamesFieldsPlaybook.java b/src/main/java/dev/dochia/cli/core/playbook/field/ZeroWidthCharsInNamesFieldsPlaybook.java
@@ -64,6 +64,11 @@ private void process(PlaybookData data, String fuzzedField, String fuzzValue) {
                 fuzzedField, FuzzingStrategy.formatValue(fuzzValue));
         testCaseListener.addExpectedResult(logger, "Should get a [{}] response code", ResponseCodeFamilyPredefined.FOURXX.asString());
 
+        if (!testCaseListener.shouldContinueExecution(logger, ResponseCodeFamilyPredefined.FOURXX)) {
+            testCaseListener.skipTest(logger, "Test skipped due to response code filtering");
+            return;
+        }
+
         HttpResponse response = serviceCaller.call(ServiceData.builder().relativePath(data.getPath()).headers(data.getHeaders())
                 .payload(fuzzedPayload).queryParams(data.getQueryParams()).httpMethod(data.getMethod()).contractPath(data.getContractPath())
                 .contentType(data.getFirstRequestContentType()).pathParamsPayload(data.getPathParamsPayload())
diff --git a/src/main/java/dev/dochia/cli/core/playbook/field/base/BaseFieldsPlaybook.java b/src/main/java/dev/dochia/cli/core/playbook/field/base/BaseFieldsPlaybook.java
@@ -134,6 +134,11 @@ protected void process(PlaybookData data, String fuzzedField, FuzzingStrategy fu
       testCaseListener.addExpectedResult(
           logger, "Should return [{}]", expectedResponseCodeBasedOnConstraints.asString());
 
+      if (!testCaseListener.shouldContinueExecution(logger, expectedResponseCodeBasedOnConstraints)) {
+        testCaseListener.skipTest(logger, "Test skipped due to response code filtering");
+        return;
+      }
+
       HttpResponse response = serviceCaller.call(serviceData);
 
       testCaseListener.reportResult(
diff --git a/src/main/java/dev/dochia/cli/core/report/TestCaseListener.java b/src/main/java/dev/dochia/cli/core/report/TestCaseListener.java
@@ -5,6 +5,7 @@
 import com.google.gson.JsonArray;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonParser;
+import dev.dochia.cli.core.args.FilterArguments;
 import dev.dochia.cli.core.args.IgnoreArguments;
 import dev.dochia.cli.core.args.ReportingArguments;
 import dev.dochia.cli.core.context.GlobalContext;
@@ -80,6 +81,7 @@ public class TestCaseListener {
     private final GlobalContext globalContext;
     private final IgnoreArguments ignoreArguments;
     private final ReportingArguments reportingArguments;
+    private final FilterArguments filterArguments;
     final List<TestCaseSummary> testCaseSummaryDetails = new ArrayList<>();
     final List<TestCaseExecutionSummary> testCaseExecutionDetails = new ArrayList<>();
 
@@ -98,16 +100,19 @@ public class TestCaseListener {
      * @param globalContext        the global context
      * @param er                   the listener for execution statistics
      * @param testReportsGenerator the generator for test reports
-     * @param filterArguments      the arguments for filtering test cases
+     * @param ignoreArguments      the arguments for ignoring results
      * @param reportingArguments   the arguments for reporting test cases
+     * @param filterArguments      the arguments for filtering results
      * @throws NoSuchElementException if no matching exporter is found for the specified report format
      */
-    public TestCaseListener(GlobalContext globalContext, ExecutionStatisticsListener er, TestReportsGenerator testReportsGenerator, IgnoreArguments filterArguments, ReportingArguments reportingArguments) {
+    public TestCaseListener(GlobalContext globalContext, ExecutionStatisticsListener er, TestReportsGenerator testReportsGenerator,
+                            IgnoreArguments ignoreArguments, ReportingArguments reportingArguments, FilterArguments filterArguments) {
         this.executionStatisticsListener = er;
         this.testReportsGenerator = testReportsGenerator;
-        this.ignoreArguments = filterArguments;
+        this.ignoreArguments = ignoreArguments;
         this.globalContext = globalContext;
         this.reportingArguments = reportingArguments;
+        this.filterArguments = filterArguments;
     }
 
     private static String replaceBrackets(String message, Object... params) {
@@ -856,18 +861,18 @@ public boolean isFieldNotADiscriminator(String fuzzedField) {
     /**
      * Returns the expected HTTP response code from the --fuzzConfig file
      *
-     * @param fuzzer       the name of the fuzzer
+     * @param playbook       the name of the playbook
      * @param defaultValue default value when property is not found
      * @param path         the current path being fuzzed
      * @param method       the current http method being fuzzed
      * @return the value of the property if found or null otherwise
      */
-    public ResponseCodeFamily getExpectedResponseCodeConfiguredFor(String fuzzer, String path, String method, ResponseCodeFamily defaultValue) {
+    public ResponseCodeFamily getExpectedResponseCodeConfiguredFor(String playbook, String path, String method, ResponseCodeFamily defaultValue) {
         List<String> keysToTry = new ArrayList<>();
-        keysToTry.add(fuzzer + "." + path + "." + method + ".expectedResponseCode");
-        keysToTry.add(fuzzer + "." + path + ".expectedResponseCode");
-        keysToTry.add(fuzzer + "." + method + ".expectedResponseCode");
-        keysToTry.add(fuzzer + ".expectedResponseCode");
+        keysToTry.add(playbook + "." + path + "." + method + ".expectedResponseCode");
+        keysToTry.add(playbook + "." + path + ".expectedResponseCode");
+        keysToTry.add(playbook + "." + method + ".expectedResponseCode");
+        keysToTry.add(playbook + ".expectedResponseCode");
 
         for (String key : keysToTry) {
             String value = globalContext.getExpectedResponseCodeConfigured(key);
@@ -886,6 +891,43 @@ public ResponseCodeFamily getExpectedResponseCodeConfiguredFor(String fuzzer, St
         return defaultValue;
     }
 
+    /**
+     * Determines if playbook execution should continue based on the expected response code.
+     * This method is used to filter playbooks when --mode NEGATIVE or --mode POSITIVE is enabled.
+     * <p>
+     * When filtering is active, this method will skip execution if the expected response code
+     * doesn't match the filter criteria and log the reason for skipping.
+     * </p>
+     *
+     * @param logger               the logger to use for skip messages
+     * @param expectedResponseCode the expected response code family for the current test
+     * @return true if execution should continue, false to skip the test
+     */
+    public boolean shouldContinueExecution(PrettyLogger logger, ResponseCodeFamily expectedResponseCode) {
+        // no filtering enabled
+        if (!filterArguments.isOnly4xxPlaybooks() && !filterArguments.isOnly2xxPlaybooks()) {
+            return true;
+        }
+
+        // can't decide -> don't filter
+        if (expectedResponseCode == null) {
+            return true;
+        }
+
+        String filterType = filterArguments.isOnly2xxPlaybooks() ? "2XX" : "4XX";
+        boolean matches = expectedResponseCode.allowedResponseCodes().stream()
+                .anyMatch(code -> ResponseCodeFamily.matchAsCodeOrRange(code, filterType));
+
+        if (!matches) {
+            logger.skip("Skipping test - expected response code {} does not match {} (--mode enabled)",
+                    expectedResponseCode.allowedResponseCodes(),
+                    filterType,
+                    filterArguments.isOnly2xxPlaybooks() ? "positive" : "negative");
+        }
+
+        return matches;
+    }
+
     private void recordResult(String message, Object[] params, String result, PrettyLogger logger) {
         TestCase testCase = currentTestCase();
         testCase.setResult(result);
diff --git a/src/test/java/dev/dochia/cli/core/playbook/field/DuplicateKeysFieldsPlaybookTest.java b/src/test/java/dev/dochia/cli/core/playbook/field/DuplicateKeysFieldsPlaybookTest.java
@@ -1,12 +1,14 @@
 package dev.dochia.cli.core.playbook.field;
 
+import dev.dochia.cli.core.args.FilterArguments;
 import dev.dochia.cli.core.http.HttpMethod;
 import dev.dochia.cli.core.http.ResponseCodeFamilyPredefined;
 import dev.dochia.cli.core.io.ServiceCaller;
 import dev.dochia.cli.core.model.HttpResponse;
 import dev.dochia.cli.core.model.PlaybookData;
 import dev.dochia.cli.core.report.TestCaseListener;
 import dev.dochia.cli.core.report.TestReportsGenerator;
+import io.github.ludovicianul.prettylogger.PrettyLogger;
 import io.quarkus.test.junit.QuarkusTest;
 import io.quarkus.test.junit.mockito.InjectSpy;
 import org.junit.jupiter.api.BeforeEach;
@@ -120,6 +122,20 @@ void shouldHandleMultipleFieldsWithLimit() {
          Mockito.verify(testCaseListener, Mockito.times(3)).reportResult(Mockito.any(), Mockito.eq(data), Mockito.eq(httpResponse), Mockito.eq(ResponseCodeFamilyPredefined.FOURXX));
      }
 
+     @Test
+     void shouldSkipTestWhen2xxFilterEnabledAndExpects4xx() {
+         FilterArguments filterArguments = Mockito.mock(FilterArguments.class);
+         Mockito.when(filterArguments.isOnly4xxPlaybooks()).thenReturn(false);
+         Mockito.when(filterArguments.isOnly2xxPlaybooks()).thenReturn(true);
+         ReflectionTestUtils.setField(testCaseListener, "filterArguments", filterArguments);
+
+         setup(HttpMethod.POST);
+         Mockito.when(data.getAllFieldsByHttpMethod()).thenReturn(Collections.singleton("field"));
+         duplicateKeysFieldsPlaybook.run(data);
+
+         Mockito.verify(testCaseListener).skipTest(Mockito.any(PrettyLogger.class), Mockito.eq("Test skipped due to response code filtering"));
+         Mockito.verify(serviceCaller, Mockito.never()).call(Mockito.any());
+     }
     private void setup(HttpMethod method) {
         httpResponse = HttpResponse.builder().body("{}").responseCode(200).build();
         Map<String, List<String>> responses = new HashMap<>();
diff --git a/src/test/java/dev/dochia/cli/core/playbook/field/NewFieldsPlaybookTest.java b/src/test/java/dev/dochia/cli/core/playbook/field/NewFieldsPlaybookTest.java
diff --git a/src/test/java/dev/dochia/cli/core/report/TestCaseListenerTest.java b/src/test/java/dev/dochia/cli/core/report/TestCaseListenerTest.java
