[sandbox] Also run the post processor in the GenerativeEngine

When the Fuzzer is in State.corpusGeneration, it will not use the
configured Fuzzer.engine but a GenerativeEngine. With this change the
GenearativeEngine will also run any registered FuzzingPostProcessor.

Change-Id: I6a1c6d262bee25edea648e369b7baf5d2a16d09c
Reviewed-on: https://chrome-internal-review.googlesource.com/c/v8/fuzzilli/+/8551317
Reviewed-by: Carl Smith <cffsmith@google.com>
Commit-Queue: Matthias Liedtke <mliedtke@google.com>
Reviewed-by: Samuel Groß <saelo@google.com>

Author: Liedtke

Sources/Fuzzilli/Engines/FuzzEngine.swift

@@ -15,7 +15,7 @@
 import Foundation
 
 public class FuzzEngine: ComponentBase {
-    private var postProcessor: FuzzingPostProcessor? = nil
+    private(set) var postProcessor: FuzzingPostProcessor? = nil
 
     override init(name: String) {
         super.init(name: name)

Sources/Fuzzilli/Fuzzer.swift

@@ -178,6 +178,11 @@ public class Fuzzer {
         self.minimizer = minimizer
         self.logger = Logger(withLabel: "Fuzzer")
 
+        // Pass-through any postprocessor to the generative engine.
+        if let postProcessor = engine.postProcessor {
+            corpusGenerationEngine.registerPostProcessor(postProcessor)
+        }
+
         // Register this fuzzer instance with its queue so that it is possible to
         // obtain a reference to the Fuzzer instance when running on its queue.
         // This creates a reference cycle, but Fuzzer instances aren't expected
@@ -804,7 +809,7 @@ public class Fuzzer {
         case .none:
             break
         case .iterationsPerformed(let maxIterations):
-            if iterations > maxIterations {
+            if iterations >= maxIterations {
                 return shutdown(reason: .finished)
             }
         case .timeFuzzed(let maxRuntime):
@@ -959,11 +964,13 @@ public class Fuzzer {
 
             switch expectedResult {
             case .shouldSucceed where execution.outcome != .succeeded:
-                logger.fatal("Testcase \"\(test)\" did not execute successfully")
+                logger.fatal("Testcase \"\(test)\" did not execute successfully" +
+                    "\nstdout:\n\(execution.stdout)\nstderr:\n\(execution.stderr)")
             case .shouldCrash where !execution.outcome.isCrash():
                 logger.fatal("Testcase \"\(test)\" did not crash")
             case .shouldNotCrash where execution.outcome.isCrash():
-                logger.fatal("Testcase \"\(test)\" unexpectedly crashed")
+                logger.fatal("Testcase \"\(test)\" unexpectedly crashed" +
+                    "\nstdout:\n\(execution.stdout)\nstderr:\n\(execution.stderr)")
             default:
                 // Test passed
                 break

Sources/Fuzzilli/Util/MockFuzzer.swift

@@ -81,9 +81,7 @@ class MockEvaluator: ProgramEvaluator {
 }
 
 /// Create a fuzzer instance usable for testing.
-public func makeMockFuzzer(config maybeConfiguration: Configuration? = nil, engine maybeEngine: FuzzEngine? = nil, runner maybeRunner: ScriptRunner? = nil, environment maybeEnvironment: JavaScriptEnvironment? = nil, evaluator maybeEvaluator: ProgramEvaluator? = nil, corpus maybeCorpus: Corpus? = nil, codeGenerators additionalCodeGenerators : [(CodeGenerator, Int)] = []) -> Fuzzer {
-    dispatchPrecondition(condition: .onQueue(DispatchQueue.main))
-
+public func makeMockFuzzer(config maybeConfiguration: Configuration? = nil, engine maybeEngine: FuzzEngine? = nil, runner maybeRunner: ScriptRunner? = nil, environment maybeEnvironment: JavaScriptEnvironment? = nil, evaluator maybeEvaluator: ProgramEvaluator? = nil, corpus maybeCorpus: Corpus? = nil, codeGenerators additionalCodeGenerators : [(CodeGenerator, Int)] = [], queue: DispatchQueue? = nil) -> Fuzzer {
     // The configuration of this fuzzer.
     let configuration = maybeConfiguration ?? Configuration(logLevel: .warning)
 
@@ -139,19 +137,29 @@ public func makeMockFuzzer(config maybeConfiguration: Configuration? = nil, engi
                         lifter: lifter,
                         corpus: corpus,
                         minimizer: minimizer,
-                        queue: DispatchQueue.main)
+                        queue: queue ?? DispatchQueue.main)
 
-    fuzzer.registerEventListener(for: fuzzer.events.Log) { ev in
-        print("[\(ev.label)] \(ev.message)")
-    }
+    let initializeFuzzer =  {
+        fuzzer.registerEventListener(for: fuzzer.events.Log) { ev in
+            print("[\(ev.label)] \(ev.message)")
+        }
 
-    fuzzer.initialize()
+        fuzzer.initialize()
 
-    // Tests can also rely on the corpus not being empty
-    let b = fuzzer.makeBuilder()
-    b.buildPrefix()
-    b.build(n: 50, by: .generating)
-    corpus.add(b.finalize(), ProgramAspects(outcome: .succeeded))
+        // Tests can also rely on the corpus not being empty
+        let b = fuzzer.makeBuilder()
+        b.buildPrefix()
+        b.build(n: 50, by: .generating)
+        corpus.add(b.finalize(), ProgramAspects(outcome: .succeeded))
+    }
+    // If a DispatchQueue was provided by the caller, initialize the fuzzer
+    // there. Otherwise initialize it directly.
+    if let queue {
+        queue.sync {initializeFuzzer()}
+    } else {
+        dispatchPrecondition(condition: .onQueue(DispatchQueue.main))
+        initializeFuzzer()
+    }
 
     return fuzzer
 }

Tests/FuzzilliTests/EngineTests.swift

@@ -0,0 +1,51 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import XCTest
+@testable import Fuzzilli
+
+class EngineTests: XCTestCase {
+    func testPostProcessorOnGenerativeEngine() throws {
+        class MockPostProcessor : FuzzingPostProcessor {
+            var callCount = 0
+            func process(_ program: Program, for fuzzer: Fuzzer) -> Program {
+                callCount += 1
+                return program
+            }
+        }
+        let mockPostProcessor = MockPostProcessor()
+
+        let engine = MutationEngine(numConsecutiveMutations: 1)
+        engine.registerPostProcessor(mockPostProcessor)
+        let q = DispatchQueue(label: "fuzzerQueue")
+        let fuzzer = makeMockFuzzer(engine: engine, queue: q)
+        XCTAssertNotNil(fuzzer.corpusGenerationEngine.postProcessor)
+        XCTAssertEqual(mockPostProcessor.callCount, 0)
+        q.sync {
+            fuzzer.start(runUntil: .iterationsPerformed(3))
+        }
+        // Synchronize on the queue 2 times, so that each time at least one new
+        // fuzzOne() is executed on the DispatchQueue in that time.
+        // This should work consistently as the DispatchQueue is not marked
+        // with DispatchQueue.Attributes.concurrent, so each time one of the
+        // que.sync {} is executed, a fuzzOne() was executed as well.
+        q.sync {}
+        q.sync {}
+        XCTAssertEqual(mockPostProcessor.callCount, 3)
+        // No more tasks are queued.
+        q.sync {}
+        XCTAssertEqual(mockPostProcessor.callCount, 3)
+        XCTAssert(fuzzer.isStopped)
+    }
+}