Move and cleanup benchmarks

Author: MaartenS11

src/test/kotlin/EmulatorTestBase.kt src/test/kotlin/DebuggerTestBase.ktsrc/test/kotlin/EmulatorTestBase.kt renamed to src/test/kotlin/DebuggerTestBase.kt

@@ -4,7 +4,7 @@ import be.ugent.topl.mio.connections.SerialConnection
 import be.ugent.topl.mio.debugger.Debugger
 import java.io.File
 
-abstract class EmulatorTestBase {
+abstract class DebuggerTestBase {
     protected val config = DebuggerConfig()
     protected val wdcliPath: String = config.wdcliPath
 
@@ -13,7 +13,7 @@ abstract class EmulatorTestBase {
     }
 
     protected fun <T> runWithDebugger(file:String, emulator: Boolean = false, action: (Debugger) -> T): T {
-        val connection = if (emulator) ProcessConnection(wdcliPath, getFile(file).path, "--no-socket") else SerialConnection(config.port)
+        val connection = if (emulator) ProcessConnection(wdcliPath, getFile(file).path, "--no-socket") else SerialConnection(config.port ?: throw RuntimeException("Port was not configured!"))
         val debugger = Debugger(connection)
         if (!emulator) {
             debugger.updateModule(getFile(file).absolutePath)

src/test/kotlin/DebuggerTests.kt

@@ -12,7 +12,7 @@ import kotlin.math.ceil
 import kotlin.random.Random
 import kotlin.test.assertEquals
 
-class DebuggerTests : EmulatorTestBase() {
+class DebuggerTests : DebuggerTestBase() {
     @Test
     fun `Test if step back results in the same state`() {
         val binaryInfo = getBinaryInfo(config.symbolicWdcliPath, getFile("blink.wasm").absolutePath)
@@ -51,6 +51,8 @@ class DebuggerTests : EmulatorTestBase() {
         }
     }
 
+    // forward-execution-checkpointing.csv
+    // First graph
     @Test
     fun `Test continue for operation speed with and without checkpointing`() {
         val writer = FileWriter(File("results-forward-execution.csv"))
@@ -253,6 +255,7 @@ class DebuggerTests : EmulatorTestBase() {
         }
     }
 
+    // step-back-reexecute.csv
     @Test
     fun `Step back test 3`() {
         val wasmFile = "prime/prime-no-mem.wasm"
@@ -383,8 +386,3 @@ class DebuggerTests : EmulatorTestBase() {
         println(compressRLE(listOf("00", "00", "00", "00", "00"), 3))
     }
 }
-
-fun main() {
-    val x = DebuggerTests()
-    x.`Test stepBack performance  2`()
-}

src/test/kotlin/RLETests.kt

@@ -5,7 +5,7 @@ import be.ugent.topl.mio.woodstate.compressRLE
 import kotlin.random.Random
 import kotlin.test.assertEquals
 
-class RLETests : EmulatorTestBase() {
+class RLETests : DebuggerTestBase() {
     private fun loadAndRestore(debugger: Debugger, mutator: (s: WOODDumpResponse) -> Unit) {
         val snapshot = debugger.snapshotFull().second
         mutator(snapshot)

src/test/kotlin/benchmarks/Benchmarks.kt

@@ -0,0 +1,113 @@
+package benchmarks
+
+import DebuggerTestBase
+import be.ugent.topl.mio.debugger.Debugger
+import getBinaryInfo
+import org.junit.jupiter.api.Test
+import java.io.File
+import java.io.FileWriter
+import java.lang.System.currentTimeMillis
+
+/**
+ * Benchmarks which use a headless version of MIO to test the performance of various operations. To get accurate
+ * performance estimates they should be performed with an actual microcontroller since communication latency is an
+ * important factor.
+ */
+class Benchmarks : DebuggerTestBase() {
+    /**
+     * Executes the code using continueFor over n instructions and measures how much time it takes with different
+     * checkpointing strategies.
+     */
+    @Test
+    fun `Measure impact of checkpointing on forward execution`() {
+        val writer = FileWriter(File("results-forward-execution.csv"))
+        writer.write("Policy, Instructions executed, Time elapsed\n")
+        val results = mutableListOf<Triple<Int, Double, Debugger.SnapshotPolicy>>()
+        for (policy in listOf(
+            Debugger.SnapshotPolicy.None(),
+            Debugger.SnapshotPolicy.Checkpointing(1),
+            Debugger.SnapshotPolicy.Checkpointing(5),
+            Debugger.SnapshotPolicy.Checkpointing(10),
+            Debugger.SnapshotPolicy.Checkpointing(50),
+            Debugger.SnapshotPolicy.Checkpointing(100))) {
+
+            for (n in 250 ..< 1500 step 250) {
+                println("Progress $n/1500")
+                var totalTime = 0L
+                val times = 10
+                repeat(times) {
+                    runWithDebugger("prime/prime-no-mem.wasm", true) {
+                        it.setSnapshotPolicy(policy)
+                        val startTime = currentTimeMillis()
+                        it.continueFor(n)
+                        totalTime += currentTimeMillis() - startTime
+                    }
+                }
+                writer.write("$policy, $n, ${totalTime.toDouble() / times}\n")
+                writer.flush()
+            }
+        }
+        writer.close()
+
+        println("Policy, Instructions executed, Time elapsed")
+        for (result in results) {
+            println("${result.third}, ${result.first}, ${result.second}")
+        }
+    }
+
+    /**
+     * Executes 1000 instructions, then steps back 1, needing 999 instructions to be re-executed. Because checkpoints
+     * are deleted, it first has to re-execute 999 then 1999 then 2999 and so on. Since it steps back one instruction it
+     * also needs to step forward.
+     */
+    @Test
+    fun `Measure re-execution speed when stepping back`() {
+        val wasmFile = "prime/prime-no-mem.wasm"
+        val binaryInfo = getBinaryInfo(config.symbolicWdcliPath, getFile(wasmFile).absolutePath)
+        val results = mutableListOf<List<Pair<Int, Long>>>()
+        repeat(10) {
+            runWithDebugger(wasmFile, true) {
+                var t = 0
+                val timings = mutableListOf<Pair<Int, Long>>()
+                it.setSnapshotPolicy(Debugger.SnapshotPolicy.Checkpointing(0xffffff))
+                it.stepInto()
+                it.checkpoints[it.checkpoints.size - 1] = null
+                timings.add(Pair(t, timeElapsed {
+                    it.stepBack(1, binaryInfo) {}
+                }))
+
+                (0..30).forEach { i ->
+                    it.continueFor(1000)
+                    it.checkpoints[it.checkpoints.size - 1] = null
+                    t += 999
+                    timings.add(Pair(t, timeElapsed {
+                        it.stepBack(1, binaryInfo) {}
+                    }))
+                    it.checkpoints[it.checkpoints.size - 1] = null
+                }
+                println(timings)
+                results.add(timings)
+            }
+        }
+        val average = mutableListOf<Pair<Int, Float>>()
+        for (i in results[0].indices) {
+            var sum = 0.0
+            for (run in results.indices) {
+                sum += results[run][i].second
+            }
+            average.add(Pair(results[0][i].first, sum.toFloat() / results.size))
+        }
+        val writer = FileWriter(File("results-step-back-re-execution.csv"))
+        writer.write("t, avg_time\n")
+        for (pair in average) {
+            writer.write("${pair.first}, ${pair.second}\n")
+        }
+        writer.close()
+    }
+
+    private fun timeElapsed(action: () -> Unit): Long {
+        val startTime = currentTimeMillis()
+        action()
+        return currentTimeMillis() - startTime
+    }
+}