Make benchmarking suite give more stable results.

Address the following issues:

- Dead Code Elimination (DCE) and Escape Analysis
- Type Profile Pollution (Megamorphic Inline Caches)
- Insufficient Warmup Duration

Author: renggli

bin/benchmark/suites/json_benchmark.dart

@@ -42,9 +42,27 @@ void runJson(String name, String input) => run(
       throw StateError('Parsers provide inconsistent results');
     }
   },
-  parse: () => jsonParser.parse(input),
-  accept: () => jsonParser.accept(input),
-  native: () => convert.json.decode(input),
+  parse: (count) {
+    var result = 0;
+    for (var c = 0; c < count; c++) {
+      result ^= jsonParser.parse(input).hashCode;
+    }
+    return result;
+  },
+  accept: (count) {
+    var result = 0;
+    for (var c = 0; c < count; c++) {
+      result ^= jsonParser.accept(input).hashCode;
+    }
+    return result;
+  },
+  native: (count) {
+    var result = 0;
+    for (var c = 0; c < count; c++) {
+      result ^= convert.json.decode(input).hashCode;
+    }
+    return result;
+  },
 );
 
 void main() {

bin/benchmark/suites/regexp_benchmark.dart

@@ -25,9 +25,27 @@ void runRegExp(String regexp, Parser<void> parser, String input) {
         throw StateError('Expressions provide inconsistent results');
       }
     },
-    parse: () => parserPattern.allMatches(input).toList(),
-    accept: () => parserPattern.allMatches(input).isNotEmpty,
-    native: () => nativePattern.allMatches(input).toList(),
+    parse: (count) {
+      var result = 0;
+      for (var c = 0; c < count; c++) {
+        result ^= parserPattern.allMatches(input).hashCode;
+      }
+      return result;
+    },
+    accept: (count) {
+      var result = 0;
+      for (var c = 0; c < count; c++) {
+        result ^= parserPattern.allMatches(input).hashCode;
+      }
+      return result;
+    },
+    native: (count) {
+      var result = 0;
+      for (var c = 0; c < count; c++) {
+        result ^= nativePattern.allMatches(input).hashCode;
+      }
+      return result;
+    },
   );
 }
 

bin/benchmark/utils/benchmark.dart

@@ -1,7 +1,7 @@
 import 'package:data/stats.dart';
 
 /// Function type to be benchmarked.
-typedef Benchmark = void Function();
+typedef Benchmark = int Function(int count);
 
 /// Measures the time it takes to run [function] in microseconds.
 ///
@@ -11,6 +11,7 @@ Jackknife<double> benchmark(
   Benchmark function, {
   int minLoop = 25,
   Duration minDuration = const Duration(milliseconds: 100),
+  Duration warmupDuration = const Duration(seconds: 1),
   int sampleCount = 25,
   double confidenceLevel = 0.95,
 }) {
@@ -25,6 +26,11 @@ Jackknife<double> benchmark(
       'in $minDuration.',
     );
   }
+  // Warmup phase.
+  final warmupWatch = Stopwatch()..start();
+  while (warmupWatch.elapsed < warmupDuration) {
+    _benchmark(function, count);
+  }
   // Collect samples.
   final samples = <double>[];
   for (var i = 0; i < sampleCount; i++) {
@@ -37,15 +43,16 @@ Jackknife<double> benchmark(
   );
 }
 
+/// Blackhole variable to prevent dead code elimination.
+int blackhole = 0;
+
 @pragma('vm:never-inline')
 @pragma('vm:unsafe:no-interrupts')
 Duration _benchmark(Benchmark function, int count) {
   final watch = Stopwatch();
   watch.start();
-  var n = count + 0;
-  while (n-- > 0) {
-    function();
-  }
+  final result = function(count);
   watch.stop();
+  blackhole ^= result;
   return watch.elapsed;
 }

bin/benchmark/utils/runner.dart

@@ -40,13 +40,13 @@ final defaultCharsInput = [
 ].shuffled(Random(42));
 final defaultStringInput = defaultCharsInput.join();
 
-final List<({String name, Benchmark benchmark})> _benchmarkEntries = (() {
+final List<({String name, void Function() benchmark})> _benchmarkEntries = (() {
   Future.delayed(const Duration(milliseconds: 1)).then((_) {
     for (final (:name, :benchmark) in _benchmarkEntries) {
       if (optionFilter == null || optionFilter == name) benchmark();
     }
   });
-  return SortedList<({String name, Benchmark benchmark})>(
+  return SortedList<({String name, void Function() benchmark})>(
     comparator: compareAsciiLowerCase.keyOf((entry) => entry.name),
   );
 })();
@@ -56,7 +56,7 @@ final numberPrinter = FixedNumberPrinter(precision: 3);
 /// Generic benchmark runner.
 void run(
   String name, {
-  required Benchmark verify,
+  required void Function() verify,
   required Benchmark parse,
   required Benchmark accept,
   Benchmark? native,
@@ -83,7 +83,7 @@ void run(
         final benchmarks = [
           benchmark(parse),
           benchmark(accept),
-          native?.also(benchmark),
+          if (native != null) benchmark(native),
         ].whereType<Jackknife<double>>();
         for (final benchmark in benchmarks) {
           stdout.write(optionSeparator);
@@ -133,15 +133,23 @@ void runChars(String name, Parser<void> parser, {int? success, String? input}) {
         throw StateError('Expected $success_ successes, but got $count');
       }
     },
-    parse: () {
-      for (var i = 0; i < inputLength; i++) {
-        parser.parse(input_, start: i);
+    parse: (count) {
+      var result = 0;
+      for (var c = 0; c < count; c++) {
+        for (var i = 0; i < inputLength; i++) {
+          result ^= parser.parse(input_, start: i).hashCode;
+        }
       }
+      return result;
     },
-    accept: () {
-      for (var i = 0; i < inputLength; i++) {
-        parser.accept(input_, start: i);
+    accept: (count) {
+      var result = 0;
+      for (var c = 0; c < count; c++) {
+        for (var i = 0; i < inputLength; i++) {
+          result ^= parser.accept(input_, start: i).hashCode;
+        }
       }
+      return result;
     },
   );
 }
@@ -172,7 +180,19 @@ void runString(
         );
       }
     },
-    parse: () => parser.parse(input_),
-    accept: () => parser.accept(input_),
+    parse: (count) {
+      var result = 0;
+      for (var c = 0; c < count; c++) {
+        result ^= parser.parse(input_).hashCode;
+      }
+      return result;
+    },
+    accept: (count) {
+      var result = 0;
+      for (var c = 0; c < count; c++) {
+        result ^= parser.accept(input_) ? 1 : 0;
+      }
+      return result;
+    },
   );
 }

bin/lisp/lisp.dart

@@ -40,7 +40,7 @@ void main(List<String> arguments) {
       } else if (option == '-i') {
         interactiveMode = true;
       } else if (option == '-?') {
-        stdout.writeln('${Platform.executable} smalltalk.dart -n -i [files]');
+        stdout.writeln('${Platform.executable} lisp.dart -n -i [files]');
         stdout.writeln(' -i enforces the interactive mode');
         stdout.writeln(' -n does not load the standard library');
         exit(0);

pubspec.yaml

@@ -31,4 +31,6 @@ dev_dependencies:
   test: ^1.26.0
 dependency_overrides:
   xml:
-    git: https://github.com/renggli/dart-xml.git
+    git: https://github.com/renggli/dart-xml.git
+  petitparser:
+    git: https://github.com/petitparser/dart-petitparser.git