Mutation runtime weighting

Scripts/RunFuzzilli.sh

@@ -0,0 +1 @@
+swift run FuzzilliCli --profile=v8 --engine=multi --corpus=postgresql --postgres-url=postgresql://fuzzilli:password@localhost:5432/fuzzilli --logLevel=verbose --timeout=1500 --diagnostics ~/projects/ritsec/vrig/vrigatoni/v8/out/fuzzbuild/d8

Scripts/SetupPostgres.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+# Setup PostgreSQL for Fuzzilli testing
+set -e
+
+echo "=== Fuzzilli PostgreSQL Setup ==="
+
+# Detect container runtime
+if command -v docker &> /dev/null && command -v docker-compose &> /dev/null; then
+    CONTAINER_RUNTIME="docker"
+    echo "Using Docker"
+elif command -v podman &> /dev/null; then
+    CONTAINER_RUNTIME="podman"
+    echo "Using Podman"
+else
+    echo "Error: Neither docker-compose nor podman is available"
+    echo "Please install docker-compose or podman to continue"
+    exit 1
+fi
+
+# Detect compose command
+if command -v docker-compose &> /dev/null; then
+    COMPOSE_CMD="docker-compose"
+elif command -v podman-compose &> /dev/null; then
+    COMPOSE_CMD="podman-compose"
+else
+    echo "Error: No compose command found, do you have docker-compose or po installed?"
+    exit 1
+fi
+
+# Check if container runtime is accessible
+if ! $CONTAINER_RUNTIME info &> /dev/null; then
+    echo "Error: $CONTAINER_RUNTIME is not accessible"
+    echo "Please ensure $CONTAINER_RUNTIME is running and try again"
+    exit 1
+fi
+
+echo "Starting PostgreSQL container..."
+$COMPOSE_CMD up -d postgres
+
+echo "Waiting for PostgreSQL to be ready..."
+timeout=60
+counter=0
+while ! $COMPOSE_CMD exec postgres pg_isready -U fuzzilli -d fuzzilli &> /dev/null; do
+    if [ $counter -ge $timeout ]; then
+        echo "Error: PostgreSQL failed to start within $timeout seconds"
+        $COMPOSE_CMD logs postgres
+        exit 1
+    fi
+    echo "Waiting for PostgreSQL... ($counter/$timeout)"
+    sleep 2
+    counter=$((counter + 2))
+done
+
+echo "PostgreSQL is ready!"
+
+# Test connection
+echo "Testing database connection..."
+$COMPOSE_CMD exec postgres psql -U fuzzilli -d fuzzilli -c "SELECT version();"
+
+echo "Checking if tables exist..."
+$COMPOSE_CMD exec postgres psql -U fuzzilli -d fuzzilli -c "
+SELECT table_name 
+FROM information_schema.tables 
+WHERE table_schema = 'public' 
+ORDER BY table_name;
+"
+
+echo "Checking execution types..."
+$COMPOSE_CMD exec postgres psql -U fuzzilli -d fuzzilli -c "
+SELECT id, title, description 
+FROM execution_type 
+ORDER BY id;
+"
+
+echo "Checking mutator types..."
+$COMPOSE_CMD exec postgres psql -U fuzzilli -d fuzzilli -c "
+SELECT id, name, category 
+FROM mutator_type 
+ORDER BY id;
+"
+
+echo "Checking execution outcomes..."
+$COMPOSE_CMD exec postgres psql -U fuzzilli -d fuzzilli -c "
+SELECT id, outcome, description 
+FROM execution_outcome 
+ORDER BY id;
+"
+
+echo ""
+echo "=== PostgreSQL Setup Complete ==="
+echo "Connection string: postgresql://fuzzilli:fuzzilli123@localhost:5433/fuzzilli"
+echo ""
+echo "To start pgAdmin (optional):"
+echo "  $COMPOSE_CMD up -d pgadmin"
+echo "  Open http://localhost:8080"
+echo "  Login: admin@fuzzilli.local / admin123"
+echo ""
+echo "To stop PostgreSQL:"
+echo "  $COMPOSE_CMD down"
+echo ""
+echo "To view logs:"
+echo "  $COMPOSE_CMD logs postgres"

Sources/Fuzzilli/Engines/FuzzEngine.swift

@@ -42,6 +42,7 @@ public class FuzzEngine: ComponentBase {
         switch execution.outcome {
             case .crashed(let termsig):
                 fuzzer.processCrash(program, withSignal: termsig, withStderr: execution.stderr, withStdout: execution.stdout, origin: .local, withExectime: execution.execTime)
+                fuzzer.runtimeWeightedMutators.adjustBatchWeight(fuzzer.runtimeWeightedMutators.getLastElements(), 1.1, 0.9)
                 program.contributors.generatedCrashingSample()
 
             case .succeeded:
@@ -57,8 +58,10 @@ public class FuzzEngine: ComponentBase {
 
                 if isInteresting {
                     program.contributors.generatedInterestingSample()
+                    fuzzer.runtimeWeightedMutators.adjustBatchWeight(fuzzer.runtimeWeightedMutators.getLastElements(), 1.1, 0.9)
                 } else {
                     program.contributors.generatedValidSample()
+                    fuzzer.runtimeWeightedMutators.adjustBatchWeight(fuzzer.runtimeWeightedMutators.getLastElements(), 0.9, 1.1)
                 }
 
             case .failed(_):

Sources/Fuzzilli/Engines/RuntimeHybridEngine.swift

@@ -0,0 +1,181 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+public class RuntimeHybridEngine: FuzzEngine {
+    // The number of mutations to perform to a single sample per round
+    private let numConsecutiveMutations: Int
+
+    // The different outcomes of one fuzzing iterations.
+    private enum CodeGenerationOutcome: String, CaseIterable {
+        case success = "Success"
+        case generatedCodeFailed = "Generated code failed"
+        case generatedCodeTimedOut = "Generated code timed out"
+        case generatedCodeCrashed = "Generated code crashed"
+    }
+    private var outcomeCounts = [CodeGenerationOutcome: Int]()
+
+    // Additional statistics about the generated programs.
+    private var totalInstructionsGenerated = 0
+    private var programsGenerated = 0
+    private var percentageOfGuardedOperationsAfterCodeGeneration = MovingAverage(n: 1000)
+    private var percentageOfGuardedOperationsAfterCodeRefining = MovingAverage(n: 1000)
+
+    // We use the FixupMutator to "fix" the generated programs based on runtime information (e.g. remove unneeded try-catch).
+    private var fixupMutator = FixupMutator(name: "HybridEngineFixupMutator")
+
+    public init(numConsecutiveMutations: Int) {
+        self.numConsecutiveMutations = numConsecutiveMutations
+        super.init(name: "HybridEngine")
+
+        for outcome in CodeGenerationOutcome.allCases {
+            outcomeCounts[outcome] = 0
+        }
+    }
+
+    override func initialize() {
+        if fuzzer.config.logLevel.isAtLeast(.verbose) {
+            fuzzer.timers.scheduleTask(every: 30 * Minutes) {
+                guard self.programsGenerated > 0 else { return }
+
+                // TODO move into Statistics?
+                self.logger.verbose("Program Template Statistics:")
+                let nameMaxLength = self.fuzzer.programTemplates.map({ $0.name.count }).max()!
+                for template in self.fuzzer.programTemplates {
+                    let name = template.name.rightPadded(toLength: nameMaxLength)
+                    let correctnessRate = Statistics.percentageOrNa(template.correctnessRate, 7)
+                    let interestingSamplesRate = Statistics.percentageOrNa(template.interestingSamplesRate, 7)
+                    let timeoutRate = Statistics.percentageOrNa(template.timeoutRate, 6)
+                    let avgInstructionsAdded = String(format: "%.2f", template.avgNumberOfInstructionsGenerated).leftPadded(toLength: 5)
+                    let samplesGenerated = template.totalSamples
+                    self.logger.verbose("    \(name) : Correctness rate: \(correctnessRate), Interesting sample rate: \(interestingSamplesRate), Timeout rate: \(timeoutRate), Avg. # of instructions generated: \(avgInstructionsAdded), Total # of generated samples: \(samplesGenerated)")
+                }
+
+                let totalOutcomes = self.outcomeCounts.values.reduce(0, +)
+                self.logger.verbose("Frequencies of code generation outcomes:")
+                for outcome in CodeGenerationOutcome.allCases {
+                    let count = self.outcomeCounts[outcome]!
+                    let frequency = (Double(count) / Double(totalOutcomes)) * 100.0
+                    self.logger.verbose("    \(outcome.rawValue.rightPadded(toLength: 25)): \(String(format: "%.2f%%", frequency))")
+                }
+
+                self.logger.verbose("Number of generated programs: \(self.programsGenerated)")
+                self.logger.verbose("Average programs size: \(self.totalInstructionsGenerated / self.programsGenerated)")
+                self.logger.verbose("Average percentage of guarded operations after code generation: \(String(format: "%.2f%", self.percentageOfGuardedOperationsAfterCodeGeneration.currentValue))%")
+                self.logger.verbose("Average percentage of guarded operations after code refining: \(String(format: "%.2f%", self.percentageOfGuardedOperationsAfterCodeRefining.currentValue))%")
+            }
+        }
+    }
+
+    private func generateTemplateProgram(template: ProgramTemplate) -> Program {
+        let b = fuzzer.makeBuilder()
+        b.traceHeader("Generating program based on \(template.name) template")
+        template.generate(in: b)
+        let program = b.finalize()
+
+        program.contributors.insert(template)
+        template.addedInstructions(program.size)
+        return program
+    }
+
+    public override func fuzzOne(_ group: DispatchGroup) {
+        let template = fuzzer.programTemplates.randomElement()
+
+        let generatedProgram = generateTemplateProgram(template: template)
+
+        // Update basic codegen statistics.
+        totalInstructionsGenerated += generatedProgram.size
+        programsGenerated += 1
+        percentageOfGuardedOperationsAfterCodeGeneration.add(computePercentageOfGuardedOperations(in: generatedProgram))
+
+        // We use a higher timeout for the initial execution as pure code generation should only rarely lead to infinite loops/recursion.
+        // On the other hand, the generated program may contain slow operations (e.g. try-catch guards) that the subsequent fixup may remove.
+        let outcome = execute(generatedProgram, withTimeout: fuzzer.config.timeout * 2)
+        switch outcome {
+        case .succeeded:
+            recordOutcome(.success)
+        case .failed:
+            return recordOutcome(.generatedCodeFailed)
+        case .timedOut:
+            return recordOutcome(.generatedCodeTimedOut)
+        case .crashed:
+            return recordOutcome(.generatedCodeCrashed)
+        }
+
+        // Now perform one round of fixup to improve the generated program based on runtime information and in particular remove all try-catch guards that are not needed.
+        // For example, at runtime we'll know the exact type of variables, including object methods and properties, which we do not necessarily know statically during code generation.
+        // As such, it is much easier to select a "good" method/property to access at runtime than it is during static code generation. Further, it is trivial to determine which
+        // operations raise an exception at runtime, but hard to determine that statically at code generation time. So we can be overly conservative and wrap many operations in
+        // try-catch (i.e. "guard" them), then remove the unnecessary guards after code generation based on runtime information. This is what fixup achieves.
+        let refinedProgram: Program
+        if let result = fixupMutator.mutate(generatedProgram, for: fuzzer) {
+            refinedProgram = result
+            percentageOfGuardedOperationsAfterCodeRefining.add(computePercentageOfGuardedOperations(in: refinedProgram))
+        } else {
+            // Fixup is expected to fail sometimes, for example if there is nothing to fix.
+            refinedProgram = generatedProgram
+        }
+
+        // Now mutate the program a number of times.
+        // We do this for example because pure code generation will often not generate "weird" code (e.g. weird inputs to operations, infinite loops, very large arrays, odd-looking object/class literals, etc.), but mutators are pretty good at that.
+        // Further, some mutators have access to runtime information (e.g. Probe and Explore mutator) which the static code generation lacks.
+        var parent = refinedProgram
+        for _ in 0..<numConsecutiveMutations {
+            // TODO: factor out code shared with the MutationEngine?
+            var mutator = fuzzer.runtimeWeightedMutators.weightedElement()
+            let maxAttempts = 10
+            var mutatedProgram: Program? = nil
+            for _ in 0..<maxAttempts {
+                if let result = mutator.mutate(parent, for: fuzzer) {
+                    // Success!
+                    result.contributors.formUnion(parent.contributors)
+                    mutator.addedInstructions(result.size - parent.size)
+                    mutatedProgram = result
+                    break
+                } else {
+                    // Try a different mutator.
+                    mutator.failedToGenerate()
+                    fuzzer.runtimeWeightedMutators.popLastElement()
+                    mutator = fuzzer.runtimeWeightedMutators.weightedElement()
+                }
+            }
+
+            guard let program = mutatedProgram else {
+                logger.warning("Could not mutate sample, giving up. Sample:\n\(FuzzILLifter().lift(parent))")
+                continue
+            }
+
+            assert(program !== parent)
+            let outcome = execute(program)
+
+            // Mutate the program further if it succeeded.
+            if .succeeded == outcome {
+                parent = program
+            }
+        }
+    }
+
+    private func recordOutcome(_ outcome: CodeGenerationOutcome) {
+        outcomeCounts[outcome]! += 1
+    }
+
+    private func computePercentageOfGuardedOperations(in program: Program) -> Double {
+        let numGuardedOperations = Double(program.code.filter({ $0.isGuarded }).count)
+        // We also count try-catch blocks as guards for the purpose of these statistics, and we count them as 3 instructions
+        // as they at least need the BeginTry and EndTryCatchFinally, plus either a BeginCatch or BeginFinally.
+        let numTryCatchBlocks = Double(program.code.filter({ $0.op is BeginTry }).count)
+        return ((numGuardedOperations + numTryCatchBlocks * 3) / Double(program.size)) * 100.0
+    }
+}

Sources/Fuzzilli/Engines/RuntimeMutationEngine.swift

@@ -0,0 +1,92 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+/// The core fuzzer responsible for generating and executing programs.
+public class RuntimeMutationEngine: FuzzEngine {
+    // The number of consecutive mutations to apply to a sample.
+    private let numConsecutiveMutations: Int
+
+    public init(numConsecutiveMutations: Int) {
+        self.numConsecutiveMutations = numConsecutiveMutations
+        super.init(name: "MutationEngine")
+    }
+
+    /// Perform one round of fuzzing.
+    ///
+    /// High-level fuzzing algorithm:
+    ///
+    ///     let parent = pickSampleFromCorpus()
+    ///     repeat N times:
+    ///         let current = mutate(parent)
+    ///         execute(current)
+    ///         if current produced crashed:
+    ///             output current
+    ///         elif current resulted in a runtime exception or a time out:
+    ///             // do nothing
+    ///         elif current produced new, interesting behaviour:
+    ///             minimize and add to corpus
+    ///         else
+    ///             parent = current
+    ///
+    ///
+    /// This ensures that samples will be mutated multiple times as long
+    /// as the intermediate results do not cause a runtime exception.
+    public override func fuzzOne(_ group: DispatchGroup) {
+        var parent = fuzzer.corpus.randomElementForMutating()
+        parent = prepareForMutating(parent)
+        for _ in 0..<numConsecutiveMutations {
+            // TODO: factor out code shared with the HybridEngine?
+            var mutator = fuzzer.runtimeWeightedMutators.weightedElement()
+            let maxAttempts = 10
+            var mutatedProgram: Program? = nil
+            for _ in 0..<maxAttempts {
+                if let result = mutator.mutate(parent, for: fuzzer) {
+                    // Success!
+                    result.contributors.formUnion(parent.contributors)
+                    mutator.addedInstructions(result.size - parent.size)
+                    mutatedProgram = result
+                    break
+                } else {
+                    // Try a different mutator.
+                    mutator.failedToGenerate()
+                    fuzzer.runtimeWeightedMutators.popLastElement()
+                    mutator = fuzzer.runtimeWeightedMutators.weightedElement()
+                }
+            }
+
+            guard let program = mutatedProgram else {
+                logger.warning("Could not mutate sample, giving up. Sample:\n\(FuzzILLifter().lift(parent))")
+                continue
+            }
+
+            assert(program !== parent)
+            let outcome = execute(program)
+
+            // Mutate the program further if it succeeded.
+            if .succeeded == outcome {
+                parent = program
+            }
+        }
+    }
+
+    /// Pre-processing of programs to facilitate mutations on them.
+    private func prepareForMutating(_ program: Program) -> Program {
+        let b = fuzzer.makeBuilder()
+        b.buildPrefix()
+        b.append(program)
+        return b.finalize()
+    }
+}