Try minimizing crashes of instrumented runs

When a program instrumented by RuntimeAssistedMutator crashes, we avoid calling
processCrash() on it immediately. This is because the boilerplate code added
during instrumentation makes such crashes difficult to minimize
(see, e.g., https://g-issues.chromium.org/issues/488963988?pli=1&authuser=0).

Instead, we follow this procedure:
1. Always log the crash of the instrumented program.
2. Check if the process()'d version of the instrumented program also crashes.
3. If yes, we call processCrash() on that program instead, as its more straightforward to minimize.

Bug: 488963988
Change-Id: Iffefc9435f4ef31a3fbf798d374a04f9f1fc115a
Reviewed-on: https://chrome-internal-review.googlesource.com/c/v8/fuzzilli/+/9129498
Reviewed-by: Matthias Liedtke <mliedtke@google.com>
Commit-Queue: Leon Bettscheider <bettscheider@google.com>

Author: lebe-g

Sources/Fuzzilli/Fuzzer.swift

@@ -789,25 +789,34 @@ public class Fuzzer {
         return true
     }
 
+    /// Collect information about a crash.
+    func collectCrashInfo(for program: Program, withSignal termsig: Int, withStderr stderr: String, withStdout stdout: String, withExectime exectime: TimeInterval) -> [String] {
+        var info = [String]()
+        info.append("CRASH INFO")
+        info.append("==========")
+        if let tag = config.tag {
+            info.append("INSTANCE TAG: \(tag)")
+        }
+        info.append("TERMSIG: \(termsig)")
+        info.append("STDERR:")
+        info.append(stderr.trimmingCharacters(in: .newlines))
+        info.append("STDOUT:")
+        info.append(stdout.trimmingCharacters(in: .newlines))
+        info.append("FUZZER ARGS: \(config.arguments.joined(separator: " "))")
+        info.append("TARGET ARGS: \(runner.processArguments.joined(separator: " "))")
+        info.append("CONTRIBUTORS: \(program.contributors.map({ $0.name }).joined(separator: ", "))")
+        info.append("EXECUTION TIME: \(Int(exectime * 1000))ms")
+        return info
+    }
+
     /// Process a program that causes a crash.
     func processCrash(_ program: Program, withSignal termsig: Int, withStderr stderr: String, withStdout stdout: String, origin: ProgramOrigin, withExectime exectime: TimeInterval) {
         func processCommon(_ program: Program) {
             let hasCrashInfo = program.comments.at(.footer)?.contains("CRASH INFO") ?? false
             if !hasCrashInfo {
-                program.comments.add("CRASH INFO", at: .footer)
-                program.comments.add("==========", at: .footer)
-                if let tag = config.tag {
-                    program.comments.add("INSTANCE TAG: \(tag)", at: .footer)
+                for line in collectCrashInfo(for: program, withSignal: termsig, withStderr: stderr, withStdout: stdout, withExectime: exectime) {
+                    program.comments.add(line, at: .footer)
                 }
-                program.comments.add("TERMSIG: \(termsig)", at: .footer)
-                program.comments.add("STDERR:", at: .footer)
-                program.comments.add(stderr.trimmingCharacters(in: .newlines), at: .footer)
-                program.comments.add("STDOUT:", at: .footer)
-                program.comments.add(stdout.trimmingCharacters(in: .newlines), at: .footer)
-                program.comments.add("FUZZER ARGS: \(config.arguments.joined(separator: " "))", at: .footer)
-                program.comments.add("TARGET ARGS: \(runner.processArguments.joined(separator: " "))", at: .footer)
-                program.comments.add("CONTRIBUTORS: \(program.contributors.map({ $0.name }).joined(separator: ", "))", at: .footer)
-                program.comments.add("EXECUTION TIME: \(Int(exectime * 1000))ms", at: .footer)
             }
             assert(program.comments.at(.footer)?.contains("CRASH INFO") ?? false)
 

Sources/Fuzzilli/Mutators/RuntimeAssistedMutator.swift

@@ -32,6 +32,7 @@ public class RuntimeAssistedMutator: Mutator {
     enum Outcome: String, CaseIterable {
         case success = "Success"
         case cannotInstrument = "Cannot instrument input"
+        case instrumentedProgramCrashed = "Instrumented program crashed"
         case instrumentedProgramFailed = "Instrumented program failed"
         case instrumentedProgramTimedOut = "Instrumented program timed out"
         case noResults = "No results received"
@@ -90,6 +91,10 @@ public class RuntimeAssistedMutator: Mutator {
 
         // Execute the instrumented program (with a higher timeout) and collect the output.
         let execution = fuzzer.execute(instrumentedProgram, withTimeout: fuzzer.config.timeout * 4, purpose: .runtimeAssistedMutation)
+        // We need to cache these because they're invalidated the next time we call execute().
+        let oldStdout = execution.stdout
+        let oldStderr = execution.stderr
+        let oldFuzzout = execution.fuzzout
         switch execution.outcome {
         case .failed(_):
             // We generally do not expect the instrumentation code itself to cause runtime exceptions. Even if it performs new actions those should be guarded with try-catch.
@@ -109,8 +114,31 @@ public class RuntimeAssistedMutator: Mutator {
             // In this case we still try to process the instrumentation output (if any) and produce the final, uninstrumented program
             // so that we obtain reliable testcase for crashes due to (1). However, to not loose crashes due to (2), we also
             // report the instrumented program as crashing here. We may therefore end up with two crashes from one mutation.
-            let stdout = execution.fuzzout + "\n" + execution.stdout
-            fuzzer.processCrash(instrumentedProgram, withSignal: signal, withStderr: execution.stderr, withStdout: stdout, origin: .local, withExectime: execution.execTime)
+
+            let stdout = oldFuzzout + "\n" + oldStdout
+
+            // We log the crash here in case something goes wrong.
+            let crashInfoText = fuzzer.collectCrashInfo(for: program, withSignal: signal, withStderr: oldStderr, withStdout: oldStdout, withExectime: execution.execTime)
+            logger.error("Instrumented program crashed: \(fuzzer.lifter.lift(program))\n\(crashInfoText.joined(separator: "\n"))")
+
+            // Check if the process()'d program also crashes. If yes, we report that crash instead.
+            // This allows to use Fuzzilli's input minimization, which wouldn't work for the instrumented program.
+            let (mutatedProgram, outcome) = process(execution.fuzzout, ofInstrumentedProgram: instrumentedProgram, using: b)
+            if let mutatedProgram {
+                assert(outcome == .success)
+
+                let execution = fuzzer.execute(mutatedProgram, withTimeout: fuzzer.config.timeout , purpose: .runtimeAssistedMutation)
+                if case .crashed(let signal) = execution.outcome {
+                    logger.info("Mutated program crashed as well, reporting mutated program instead")
+                    let stdout = execution.fuzzout + "\n" + execution.stdout
+                    fuzzer.processCrash(mutatedProgram, withSignal: signal, withStderr: execution.stderr, withStdout: stdout, origin: .local, withExectime: execution.execTime)
+                    return failure(.instrumentedProgramCrashed)
+                }
+            }
+
+            // If we reach here, the process()'d program did not crash, so we need to report the instrumented program.
+            logger.info("Mutated program did not crash, reporting original crash of the instrumented program")
+            fuzzer.processCrash(instrumentedProgram, withSignal: signal, withStderr: oldStderr, withStdout: stdout, origin: .local, withExectime: execution.execTime)
         case .succeeded:
             // The expected case.
             break
@@ -119,7 +147,7 @@ public class RuntimeAssistedMutator: Mutator {
         }
 
         // Process the output to build the mutated program.
-        let (maybeMutatedProgram, outcome) = process(execution.fuzzout, ofInstrumentedProgram: instrumentedProgram, using: b)
+        let (maybeMutatedProgram, outcome) = process(oldFuzzout, ofInstrumentedProgram: instrumentedProgram, using: b)
         guard let mutatedProgram = maybeMutatedProgram else {
             assert(outcome != .success)
             return failure(outcome)

Tests/FuzzilliTests/CrashingInstrumentationMutator.swift

@@ -0,0 +1,55 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+@testable import Fuzzilli
+
+// This mutator generates a crashing instrumented program.
+// It is used for testing: the process()'d program should be
+// reported instead of the instrument()'d program, if that program
+// also crashes, in order to enable better minimization.
+
+class CrashingInstrumentationMutator: RuntimeAssistedMutator {
+    private let shouldProcessedProgramCrash: Bool
+
+    init(shouldProcessedProgramCrash: Bool = true) {
+        self.shouldProcessedProgramCrash = shouldProcessedProgramCrash
+        super.init("CrashingInstrumentationMutator", verbose: true)
+    }
+
+    override func instrument(_ program: Program, for fuzzer: Fuzzer) -> Program? {
+        let b = fuzzer.makeBuilder()
+        b.eval("fuzzilli('FUZZILLI_CRASH', 0)");
+
+        // Add a JsInternalOperation to satisfy the assertion in RuntimeAssistedMutator.swift:89
+        let v = b.loadInt(42)
+        b.doPrint(v)
+
+        b.append(program)
+        return b.finalize()
+    }
+
+
+    override func process(_ output: String, ofInstrumentedProgram instrumentedProgram: Program, using b: ProgramBuilder) -> (Program?, Outcome) {
+        // Purpose of this print: Distinguish crashes originating from instrument() and process().
+        let printFct = b.createNamedVariable(forBuiltin: "print")
+        b.callFunction(printFct, withArgs: [b.loadString("This is the processed program")])
+        if self.shouldProcessedProgramCrash {
+            b.append(instrumentedProgram)
+        }
+        return (b.finalize(), .success)
+    }
+
+    override func logAdditionalStatistics() {}
+}

Tests/FuzzilliTests/RuntimeAssistedMutatorTests.swift

@@ -0,0 +1,82 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import XCTest
+@testable import Fuzzilli
+
+class RuntimeAssistedMutatorTests: XCTestCase {
+    class CrashMockScriptRunner: ScriptRunner {
+        var processArguments: [String] = []
+        var env: [(String, String)] = []
+        func run(_ script: String, withTimeout timeout: UInt32) -> Execution {
+            if script.contains("fuzzilli('FUZZILLI_CRASH', 0)") {
+                return MockExecution(outcome: .crashed(9), stdout: "", stderr: "", fuzzout: "", execTime: 0.1)
+            } else {
+                return MockExecution(outcome: .succeeded, stdout: "", stderr: "", fuzzout: "", execTime: 0.1)
+            }
+        }
+        func setEnvironmentVariable(_ key: String, to value: String) {}
+        func initialize(with fuzzer: Fuzzer) {}
+        var isInitialized: Bool { true }
+    }
+
+    // This test checks that if *both* the instrumented and the processed programs crash,
+    // we report the processed program.
+
+    func testInstrumentedAndProcessedProgramsCrash() {
+        let runner = CrashMockScriptRunner()
+        let config = Configuration(logLevel: .error)
+        let fuzzer = makeMockFuzzer(config: config, runner: runner)
+
+        let mutator = CrashingInstrumentationMutator()
+        let b = fuzzer.makeBuilder()
+        b.loadInt(42)
+        let program = b.finalize()
+
+        let crashEventTriggered = self.expectation(description: "Crash reported on processed program")
+        fuzzer.registerEventListener(for: fuzzer.events.CrashFound) { ev in
+            let lifted = fuzzer.lifter.lift(ev.program)
+            if lifted.contains("This is the processed program") {
+                crashEventTriggered.fulfill()
+            }
+        }
+
+        _ = mutator.mutate(program, using: b, for: fuzzer)
+        waitForExpectations(timeout: 5, handler: nil)
+    }
+
+    // This test checks that if *only* the instrumented program crashes, we report it.
+
+    func testOnlyInstrumentedProgramCrashes() {
+        let runner = CrashMockScriptRunner()
+        let config = Configuration(logLevel: .error)
+        let fuzzer = makeMockFuzzer(config: config, runner: runner)
+
+        let mutator = CrashingInstrumentationMutator(shouldProcessedProgramCrash: false)
+        let b = fuzzer.makeBuilder()
+        b.loadInt(42)
+        let program = b.finalize()
+
+        let crashEventTriggered = self.expectation(description: "Crash reported on instrumented program")
+        fuzzer.registerEventListener(for: fuzzer.events.CrashFound) { ev in
+            let lifted = fuzzer.lifter.lift(ev.program)
+            if !lifted.contains("This is the processed program") {
+                crashEventTriggered.fulfill()
+            }
+        }
+
+        _ = mutator.mutate(program, using: b, for: fuzzer)
+        waitForExpectations(timeout: 5, handler: nil)
+    }
+}