Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

MAB for Mutator selection only #348

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

MAB for Mutator selection only #348

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Sources/Fuzzilli/Core/Events.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,9 @@ public class Events {

/// Signals that a worker has disconnected.
public let WorkerDisconnected = Event<UUID>()

/// Signals when an iteration has completed.
public let IterationComplete = Event<Void>()
}

/// Reasons for shutting down a fuzzer instance.
Expand Down
6 changes: 5 additions & 1 deletion Sources/Fuzzilli/Core/FuzzEngine.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@ extension FuzzEngine {

switch execution.outcome {
case .crashed(let termsig):
fuzzer.processCrash(program, withSignal: termsig, withStderr: execution.stderr, origin: .local)
fuzzer.processCrash(program, withSignal: termsig, withStderr: execution.stderr, origin: .local)
let newCoverage = fuzzer.evaluator.newCoverageFound
fuzzer.evaluateSuccessForSelectedMutator(newCoverageFound: newCoverage)

case .succeeded:
fuzzer.dispatchEvent(fuzzer.events.ValidProgramFound, data: program)
Expand All @@ -39,6 +41,8 @@ extension FuzzEngine {
program.comments.add("Program is interesting due to \(aspects)", at: .footer)
}
fuzzer.processInteresting(program, havingAspects: aspects, origin: .local)
let newCoverage = fuzzer.evaluator.newCoverageFound
fuzzer.evaluateSuccessForSelectedMutator(newCoverageFound: newCoverage)
}
stats.producedValidSample()

Expand Down
11 changes: 8 additions & 3 deletions Sources/Fuzzilli/Core/MutationEngine.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,18 +120,20 @@ public class MutationEngine: ComponentBase, FuzzEngine {
public func fuzzOne(_ group: DispatchGroup) {
var parent = prepareForMutation(fuzzer.corpus.randomElementForMutating())
var program = parent

for _ in 0..<numConsecutiveMutations {
var mutator = fuzzer.mutators.randomElement()
var mutator = fuzzer.selectRandomMutator()
var mutated = false
for _ in 0..<10 {

if let result = mutator.mutate(parent, for: fuzzer) {
program = result
mutated = true
break
}
logger.verbose("\(mutator.name) failed, trying different mutator")
mutator = fuzzer.mutators.randomElement()

mutator = fuzzer.selectRandomMutator()
}

if !mutated {
Expand All @@ -146,6 +148,9 @@ public class MutationEngine: ComponentBase, FuzzEngine {
parent = program
}
}

// Inform mutator MAB that mutation accumulation has finished and new coverage found can be evaluated
fuzzer.notifySimultaneousMutationsComplete()
}

/// Set program prefix, should be used only in tests
Expand Down
8 changes: 7 additions & 1 deletion Sources/Fuzzilli/Evaluation/ProgramCoverageEvaluator.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,11 @@ public class ProgramCoverageEvaluator: ComponentBase, ProgramEvaluator {
public var currentScore: Double {
return Double(context.found_edges) / Double(context.num_edges)
}

/// The latest edge coverage percentage
public var newCoverageFound: Double {
return Double(context.new_found_edges) / Double(context.num_edges)
}

/// Context for the C library.
private var context = libcoverage.cov_context()
Expand Down Expand Up @@ -273,7 +278,8 @@ public class ProgramCoverageEvaluator: ComponentBase, ProgramEvaluator {
}

context.found_edges = foundEdges

context.new_found_edges = foundEdges

var start = state.startIndex + 24
state.copyBytes(to: context.virgin_bits, from: start..<start + Int(bitmapSize))
start += Int(bitmapSize)
Expand Down
3 changes: 3 additions & 0 deletions Sources/Fuzzilli/Evaluation/ProgramEvaluator.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ public protocol ProgramEvaluator: Component {

/// The current, accumulated score of all seen samples. E.g. total coverage.
var currentScore: Double { get }

/// The coverage percentage for latest new edges found
var newCoverageFound: Double { get }

/// Export the current state of this evaluator so it can be replicated.
func exportState() -> Data
Expand Down
1 change: 1 addition & 0 deletions Sources/Fuzzilli/Execution/REPRL.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ class REPRLExecution: Execution {
private let execId: Int

var outcome = ExecutionOutcome.succeeded
var coverageFound: Double = 0.0
var execTime: TimeInterval = 0

init(from reprl: REPRL) {
Expand Down
114 changes: 106 additions & 8 deletions Sources/Fuzzilli/Fuzzer.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,10 @@ public class Fuzzer {
public let programTemplates: WeightedList<ProgramTemplate>

/// The mutators used by the engine.
public let mutators: WeightedList<Mutator>
public var mutators: WeightedList<Mutator>

/// MAB for mutators
private var mabMutator: MutatorMultiArmedBandit

/// The evaluator to score generated programs.
public let evaluator: ProgramEvaluator
Expand Down Expand Up @@ -138,6 +141,7 @@ public class Fuzzer {
self.runner = scriptRunner
self.minimizer = minimizer
self.logger = Logger(withLabel: "Fuzzer")
self.mabMutator = MutatorMultiArmedBandit(actions: mutators)

// Register this fuzzer instance with its queue so that it is possible to
// obtain a reference to the Fuzzer instance when running on its queue.
Expand All @@ -146,6 +150,7 @@ public class Fuzzer {
self.queue.setSpecific(key: Fuzzer.dispatchQueueKey, value: self)
}


/// Returns the fuzzer for the active DispatchQueue.
public static var current: Fuzzer? {
return DispatchQueue.getSpecific(key: Fuzzer.dispatchQueueKey)
Expand Down Expand Up @@ -415,14 +420,25 @@ public class Fuzzer {
dispatchPrecondition(condition: .onQueue(queue))

if supportsFastStateSynchronization {
let state = try Fuzzilli_Protobuf_FuzzerState.with {
let state = try Fuzzilli_Protobuf_FuzzerStateWithFastSync.with {
$0.corpus = try corpus.exportState()
$0.evaluatorState = evaluator.exportState()
$0.mabState = self.exportMABState()
}
return try state.serializedData()
} else {
// Just export all samples in the current corpus
return try encodeProtobufCorpus(exportCorpus())
let state = try Fuzzilli_Protobuf_FuzzerState.with {
$0.corpus = try encodeProtobufCorpus(exportCorpus())
$0.mabState = self.exportMABState()
}
return try state.serializedData()
}
}

/// Returns the mab states of the current fuzzer instance
public func exportMABState() -> Fuzzilli_Protobuf_MABState {
return Fuzzilli_Protobuf_MABState.with {
$0.mutatorWeights = mabMutator.exportState()
}
}

Expand All @@ -431,15 +447,22 @@ public class Fuzzer {
dispatchPrecondition(condition: .onQueue(queue))

if supportsFastStateSynchronization {
let state = try Fuzzilli_Protobuf_FuzzerState(serializedData: data)
let state = try Fuzzilli_Protobuf_FuzzerStateWithFastSync(serializedData: data)
try corpus.importState(state.corpus)
try evaluator.importState(state.evaluatorState)
mabMutator.importState(from: state.mabState.mutatorWeights)
} else {
let corpus = try decodeProtobufCorpus(data)
let state = try Fuzzilli_Protobuf_FuzzerState(serializedData: data)
let corpus = try decodeProtobufCorpus(state.corpus)
importCorpus(corpus, importMode: .interestingOnly(shouldMinimize: false))
mabMutator.importState(from: state.mabState.mutatorWeights)
}
}

public func importMABState(from state: Fuzzilli_Protobuf_MABState) {
mabMutator.importState(from: state.mutatorWeights)
}

/// Whether the internal state of this fuzzer instance can be serialized and restored elsewhere, e.g. on a worker instance.
private var supportsFastStateSynchronization: Bool {
// We might eventually need to check that the other relevant components
Expand Down Expand Up @@ -649,6 +672,80 @@ public class Fuzzer {
return ProgramBuilder(for: self, parent: parent, interpreter: interpreter, mode: mode)
}

///
/// Functions to handle Mutator MAB
///

/// Returns a random mutator based on the MAB calculated probability
public func selectRandomMutator() -> Mutator {
// If we are under the minimum number of iterations then use mode other
// This is being done since mutators have pre determined weights on mab initialisation
if iterations < mabMutator.critMassThreshold {
return mabMutator.randomElement(mode:.other)
} else {
return mabMutator.randomElement(mode:.epoch)
}
}

/// Update the count for a successful mutator invocation
public func evaluateSuccessForSelectedMutator(newCoverageFound: Double) {
self.mabMutator.evaluateMutationSuccess(newCoverageFound: newCoverageFound)
}

/// MAB weights, estimated reward and trial updates for Mutators
public func notifySimultaneousMutationsComplete() {
/// Check if we should hard restart MAB
if mabMutator.restartThresholdReached(iterations) {
mabMutator.restartMAB()
} else if mabMutator.critMassIterationsReached(iterations) {
// estimated reward is less than our guessed upperbound
if mabMutator.epochReached() {
// estimated reward is greater than our upperbound, update the epoch count
mabMutator.epochCountUpdate()
//As we are in a new epoch reset the mutator with the max total estimated reward
mabMutator.resetMaxEstimatedTotalReward()
} else {
// update weighted action rewards and invocation counts
self.mabMutator.evaluateTotal(iterations: iterations)

// Update mutator weights
self.mabMutator.updateWeightedActionsIfEpochNotReached()
// Update trials and Total estimated rewards if epoch not reached
self.mabMutator.updateTotalEstimatedRewardIfEpochNotReached()
self.mabMutator.updateTrialsIfEpochNotReached()

/// Reset Mutator Count and Coverage Tracker
self.mabMutator.resetSimultaneousMutationTracker()
}

// If we reach a critical mass of iterations but not an epoch then
// Rescale weights so that we don't encounter crazy run offs
mabMutator.rescaleWeights()
}
}

/// Stat printing for mutator mab
/// Updated after every critical mass threshold iterations have elapsed
public func getMABMutatorStats() -> String {
return """
Mutator MAB Statistics:
----------------------
Iteration: \(self.iterations)
Trial: \(self.mabMutator.trials)
Epoch: \(self.mabMutator.epochs)
EpochThreshold: \(self.mabMutator.epochThreshold)
Gamma: \(self.mabMutator.gamma)
Critical Mass Threshold: \(self.mabMutator.critMassThreshold)
Coverage Score(MAB): \(self.mabMutator.coverageScoreMAB())
\(self.mabMutator.toString())
"""
}

// Stats as JSON to be written to disk
public func getMABMutatorStatsAsJSON() -> String {
return "{\"Iteration\": \(self.iterations), \"Trial\": \(self.mabMutator.trials), \"Epoch\": \(self.mabMutator.epochs), \"EpochThreshold\": \(self.mabMutator.epochThreshold), \"Gamma\": \(self.mabMutator.gamma), \(self.mabMutator.toJSON())}"
}

/// Performs one round of fuzzing.
private func fuzzOne() {
dispatchPrecondition(condition: .onQueue(queue))
Expand All @@ -660,9 +757,10 @@ public class Fuzzer {
return shutdown(reason: .finished)
}
iterations += 1

engine.fuzzOne(fuzzGroup)


dispatchEvent(events.IterationComplete)

// Do the next fuzzing iteration as soon as all tasks related to the current iteration are finished.
fuzzGroup.notify(queue: queue) {
self.fuzzOne()
Expand Down
111 changes: 111 additions & 0 deletions Sources/Fuzzilli/Protobuf/operations.pb.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1502,6 +1502,117 @@ public struct Fuzzilli_Protobuf_Nop {
public init() {}
}

#if swift(>=5.5) && canImport(_Concurrency)
extension Fuzzilli_Protobuf_UnaryOperator: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BinaryOperator: @unchecked Sendable {}
extension Fuzzilli_Protobuf_Comparator: @unchecked Sendable {}
extension Fuzzilli_Protobuf_LoadInteger: @unchecked Sendable {}
extension Fuzzilli_Protobuf_LoadBigInt: @unchecked Sendable {}
extension Fuzzilli_Protobuf_LoadFloat: @unchecked Sendable {}
extension Fuzzilli_Protobuf_LoadString: @unchecked Sendable {}
extension Fuzzilli_Protobuf_LoadBoolean: @unchecked Sendable {}
extension Fuzzilli_Protobuf_LoadUndefined: @unchecked Sendable {}
extension Fuzzilli_Protobuf_LoadNull: @unchecked Sendable {}
extension Fuzzilli_Protobuf_LoadThis: @unchecked Sendable {}
extension Fuzzilli_Protobuf_LoadArguments: @unchecked Sendable {}
extension Fuzzilli_Protobuf_LoadRegExp: @unchecked Sendable {}
extension Fuzzilli_Protobuf_CreateObject: @unchecked Sendable {}
extension Fuzzilli_Protobuf_CreateArray: @unchecked Sendable {}
extension Fuzzilli_Protobuf_CreateTemplateString: @unchecked Sendable {}
extension Fuzzilli_Protobuf_CreateObjectWithSpread: @unchecked Sendable {}
extension Fuzzilli_Protobuf_CreateArrayWithSpread: @unchecked Sendable {}
extension Fuzzilli_Protobuf_LoadBuiltin: @unchecked Sendable {}
extension Fuzzilli_Protobuf_LoadProperty: @unchecked Sendable {}
extension Fuzzilli_Protobuf_StoreProperty: @unchecked Sendable {}
extension Fuzzilli_Protobuf_StorePropertyWithBinop: @unchecked Sendable {}
extension Fuzzilli_Protobuf_DeleteProperty: @unchecked Sendable {}
extension Fuzzilli_Protobuf_LoadElement: @unchecked Sendable {}
extension Fuzzilli_Protobuf_StoreElement: @unchecked Sendable {}
extension Fuzzilli_Protobuf_StoreElementWithBinop: @unchecked Sendable {}
extension Fuzzilli_Protobuf_DeleteElement: @unchecked Sendable {}
extension Fuzzilli_Protobuf_LoadComputedProperty: @unchecked Sendable {}
extension Fuzzilli_Protobuf_StoreComputedProperty: @unchecked Sendable {}
extension Fuzzilli_Protobuf_StoreComputedPropertyWithBinop: @unchecked Sendable {}
extension Fuzzilli_Protobuf_DeleteComputedProperty: @unchecked Sendable {}
extension Fuzzilli_Protobuf_TypeOf: @unchecked Sendable {}
extension Fuzzilli_Protobuf_InstanceOf: @unchecked Sendable {}
extension Fuzzilli_Protobuf_In: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BeginPlainFunctionDefinition: @unchecked Sendable {}
extension Fuzzilli_Protobuf_EndPlainFunctionDefinition: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BeginArrowFunctionDefinition: @unchecked Sendable {}
extension Fuzzilli_Protobuf_EndArrowFunctionDefinition: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BeginGeneratorFunctionDefinition: @unchecked Sendable {}
extension Fuzzilli_Protobuf_EndGeneratorFunctionDefinition: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BeginAsyncFunctionDefinition: @unchecked Sendable {}
extension Fuzzilli_Protobuf_EndAsyncFunctionDefinition: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BeginAsyncArrowFunctionDefinition: @unchecked Sendable {}
extension Fuzzilli_Protobuf_EndAsyncArrowFunctionDefinition: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BeginAsyncGeneratorFunctionDefinition: @unchecked Sendable {}
extension Fuzzilli_Protobuf_EndAsyncGeneratorFunctionDefinition: @unchecked Sendable {}
extension Fuzzilli_Protobuf_Return: @unchecked Sendable {}
extension Fuzzilli_Protobuf_Yield: @unchecked Sendable {}
extension Fuzzilli_Protobuf_YieldEach: @unchecked Sendable {}
extension Fuzzilli_Protobuf_Await: @unchecked Sendable {}
extension Fuzzilli_Protobuf_CallMethod: @unchecked Sendable {}
extension Fuzzilli_Protobuf_CallComputedMethod: @unchecked Sendable {}
extension Fuzzilli_Protobuf_CallFunction: @unchecked Sendable {}
extension Fuzzilli_Protobuf_Construct: @unchecked Sendable {}
extension Fuzzilli_Protobuf_UnaryOperation: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BinaryOperation: @unchecked Sendable {}
extension Fuzzilli_Protobuf_ReassignWithBinop: @unchecked Sendable {}
extension Fuzzilli_Protobuf_Dup: @unchecked Sendable {}
extension Fuzzilli_Protobuf_Reassign: @unchecked Sendable {}
extension Fuzzilli_Protobuf_DestructArray: @unchecked Sendable {}
extension Fuzzilli_Protobuf_DestructArrayAndReassign: @unchecked Sendable {}
extension Fuzzilli_Protobuf_DestructObject: @unchecked Sendable {}
extension Fuzzilli_Protobuf_DestructObjectAndReassign: @unchecked Sendable {}
extension Fuzzilli_Protobuf_Compare: @unchecked Sendable {}
extension Fuzzilli_Protobuf_ConditionalOperation: @unchecked Sendable {}
extension Fuzzilli_Protobuf_Eval: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BeginClassDefinition: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BeginMethodDefinition: @unchecked Sendable {}
extension Fuzzilli_Protobuf_EndClassDefinition: @unchecked Sendable {}
extension Fuzzilli_Protobuf_CallSuperConstructor: @unchecked Sendable {}
extension Fuzzilli_Protobuf_CallSuperMethod: @unchecked Sendable {}
extension Fuzzilli_Protobuf_LoadSuperProperty: @unchecked Sendable {}
extension Fuzzilli_Protobuf_StoreSuperProperty: @unchecked Sendable {}
extension Fuzzilli_Protobuf_StoreSuperPropertyWithBinop: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BeginWith: @unchecked Sendable {}
extension Fuzzilli_Protobuf_EndWith: @unchecked Sendable {}
extension Fuzzilli_Protobuf_LoadFromScope: @unchecked Sendable {}
extension Fuzzilli_Protobuf_StoreToScope: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BeginIf: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BeginElse: @unchecked Sendable {}
extension Fuzzilli_Protobuf_EndIf: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BeginSwitch: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BeginSwitchCase: @unchecked Sendable {}
extension Fuzzilli_Protobuf_SwitchBreak: @unchecked Sendable {}
extension Fuzzilli_Protobuf_EndSwitch: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BeginWhile: @unchecked Sendable {}
extension Fuzzilli_Protobuf_EndWhile: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BeginDoWhile: @unchecked Sendable {}
extension Fuzzilli_Protobuf_EndDoWhile: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BeginFor: @unchecked Sendable {}
extension Fuzzilli_Protobuf_EndFor: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BeginForIn: @unchecked Sendable {}
extension Fuzzilli_Protobuf_EndForIn: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BeginForOf: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BeginForOfWithDestruct: @unchecked Sendable {}
extension Fuzzilli_Protobuf_EndForOf: @unchecked Sendable {}
extension Fuzzilli_Protobuf_LoopBreak: @unchecked Sendable {}
extension Fuzzilli_Protobuf_Continue: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BeginTry: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BeginCatch: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BeginFinally: @unchecked Sendable {}
extension Fuzzilli_Protobuf_EndTryCatch: @unchecked Sendable {}
extension Fuzzilli_Protobuf_ThrowException: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BeginCodeString: @unchecked Sendable {}
extension Fuzzilli_Protobuf_EndCodeString: @unchecked Sendable {}
extension Fuzzilli_Protobuf_BeginBlockStatement: @unchecked Sendable {}
extension Fuzzilli_Protobuf_EndBlockStatement: @unchecked Sendable {}
extension Fuzzilli_Protobuf_Nop: @unchecked Sendable {}
#endif // swift(>=5.5) && canImport(_Concurrency)

// MARK: - Code below here is support for the SwiftProtobuf runtime.

fileprivate let _protobuf_package = "fuzzilli.protobuf"
Expand Down
Loading