Added some basic microbenchmarks for the workflow runtime.

These will be expanded on to measure the Compose runtime implementation
in #1442.

Author: zach-klippenstein

.github/workflows/kotlin.yml

@@ -610,8 +610,8 @@ jobs:
           report_paths: '**/build/test-results/*[tT]est/TEST-*.xml'
           annotate_only: true # no write check perms on this workflow right now.
 
-  performance-tests:
-    name: Performance tests
+  macrobenchmarks:
+    name: Macrobenchmarks
     runs-on: workflow-kotlin-test-runner-ubuntu-4core
     timeout-minutes: 45
     strategy:
@@ -627,13 +627,43 @@ jobs:
       - name: Instrumented tests
         uses: ./.github/actions/gradle-tasks-with-emulator
         with:
-          tests-name: perf-tests-results
+          tests-name: macrobenchmarks-results
           api-level: ${{ matrix.api-level }}
           prepare-task: :benchmarks:performance-poetry:complex-poetry:prepareDebugAndroidTestArtifacts
           test-task: :benchmarks:performance-poetry:complex-poetry:connectedCheck --continue
           restore-cache-key: androidTest-build-artifacts
           failure-path-upload: '**/build/reports/androidTests/connected'
-          failure-upload-name: 'perf-tests-report'
+          failure-upload-name: 'macrobenchmarks-report'
+
+  microbenchmarks:
+    name: Microbenchmarks
+    runs-on: workflow-kotlin-test-runner-ubuntu-4core
+    timeout-minutes: 45
+    strategy:
+      # Allow tests to continue on other devices if they fail on one device.
+      fail-fast: false
+      matrix:
+        api-level:
+          - 31
+    steps:
+      - name: Checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Instrumented tests
+        uses: ./.github/actions/gradle-tasks-with-emulator
+        with:
+          tests-name: microbenchmarks-results
+          api-level: ${{ matrix.api-level }}
+          # Microbenchmarks only run as release builds so we don't use the debug task like most
+          # builds.
+          prepare-task: :benchmarks:runtime-microbenchmark:prepareReleaseAndroidTestArtifacts -Pandroidx.benchmark.dryRunMode.enable=true -Pkotlin.native.ignoreDisabledTargets=true
+          # Run in dry run mode so they only run once, without warmup, since we don't care about measurements.
+          # See https://developer.android.com/topic/performance/benchmarking/microbenchmark-instrumentation-args#dryrunmode-enable
+          # Ignore iOS and other native targets being disabled since this is specifically and Android task.
+          test-task: :benchmarks:runtime-microbenchmark:connectedReleaseAndroidTest -Pandroidx.benchmark.dryRunMode.enable=true -Pkotlin.native.ignoreDisabledTargets=true --continue
+          restore-cache-key: androidTest-build-artifacts
+          failure-path-upload: 'benchmarks/runtime-microbenchmark/build/reports/androidTests/connected'
+          failure-upload-name: 'microbenchmarks-report'
 
   instrumentation-tests:
     name: Instrumentation tests
@@ -659,7 +689,7 @@ jobs:
           api-level: ${{ matrix.api-level }}
           prepare-task: prepareConnectedCheckShard${{matrix.shardNum}}
           # -x excludes tasks
-          test-task: connectedCheckShard${{matrix.shardNum}} -x :benchmarks:dungeon-benchmark:connectedAndroidTest -x :benchmarks:performance-poetry:complex-benchmark:connectedAndroidTest -x :benchmarks:performance-poetry:complex-poetry:connectedAndroidTest
+          test-task: connectedCheckShard${{matrix.shardNum}} -x :benchmarks:dungeon-benchmark:connectedAndroidTest -x :benchmarks:performance-poetry:complex-benchmark:connectedAndroidTest -x :benchmarks:performance-poetry:complex-poetry:connectedAndroidTest -x :benchmarks:runtime-microbenchmark:connectedAndroidTest
           write-cache-key: androidTest-build-artifacts-${{matrix.shardNum}}
           restore-cache-key: main-build-artifacts
           failure-path-upload: '**/build/reports/androidTests/connected'
@@ -690,7 +720,7 @@ jobs:
           api-level: ${{ matrix.api-level }}
           prepare-task: prepareConnectedCheckShard${{matrix.shardNum}} -Pworkflow.runtime=${{matrix.runtime}}
           # -x excludes tasks
-          test-task: connectedCheckShard${{matrix.shardNum}} -Pworkflow.runtime=${{matrix.runtime}} -x :benchmarks:dungeon-benchmark:connectedAndroidTest -x :benchmarks:performance-poetry:complex-benchmark:connectedAndroidTest -x :benchmarks:performance-poetry:complex-poetry:connectedAndroidTest
+          test-task: connectedCheckShard${{matrix.shardNum}} -Pworkflow.runtime=${{matrix.runtime}} -x :benchmarks:dungeon-benchmark:connectedAndroidTest -x :benchmarks:performance-poetry:complex-benchmark:connectedAndroidTest -x :benchmarks:performance-poetry:complex-poetry:connectedAndroidTest -x :benchmarks:runtime-microbenchmark:connectedAndroidTest
           write-cache-key: androidTest-build-artifacts-${{matrix.shardNum}}-${{matrix.runtime}}
           restore-cache-key: main-build-artifacts
           failure-path-upload: '**/build/reports/androidTests/connected'
@@ -724,7 +754,8 @@ jobs:
       - unit-conflate-partial-drainExclusive-runtime-tests
       - unit-all-runtime-tests
       - ktlint
-      - performance-tests
+      - macrobenchmarks
+      - microbenchmarks
       - runtime-instrumentation-tests
       - shards-and-version
       - tutorials

benchmarks/README.md

@@ -5,7 +5,18 @@ deterministic tests - such as those for render passes - can be run on any device
 otherwise), but the benchmarks should be run on physical devices, or the closest approximation
 to physical devices you can get.
 
-## Baseline Profiles
+There are two types of benchmarks:
+
+- **Macro benchmarks** measure a particular flow in an entire app and are used to generate baseline
+  profiles. They're analogous to integration tests.
+- **Micro benchmarks** measure a specific operation or set of small operations but running them
+  many times and collecting aggregate statistics. They're used to compare detailed performance of
+  competing implementations, detect specific performance regressions, and inform optimizations.
+  They're analogous to unit tests.
+
+## Macro benchmarks
+
+### Baseline Profiles
 
 The sample apps can be used to extract 'baseline profiles' to improve code loading time after first
 install. See [baseline profiles](https://developer.android.com/studio/profile/baselineprofiles).
@@ -36,7 +47,7 @@ This will create an output file separated by module and then also by package as
 profile for each module can be added into its /src/main directory as `baseline-prof.txt`. Then on a
 release build this will be included with the resulting APK/binary.
 
-## dungeon-benchmark
+### dungeon-benchmark
 
 These are benchmarks for the [../samples/dungeon] app. Please instead use performance-poetry where
 possible.
@@ -57,11 +68,11 @@ same scenario with and without forcing the use of the profiles. To force the use
 `libs.androidx.profileinstaller` dependency is included into the app under profile (dungeon in this
 case) for side-loading the profiles.
 
-## performance-poetry
+### performance-poetry
 
 Module of code for performance testing related to poetry applications.
 
-### complex-poetry
+#### complex-poetry
 
 This application is a modified version of the samples/containers/app-poetry app which also uses the
 common components in samples/containers/common and samples/containers/poetry. It modifies this
@@ -92,9 +103,23 @@ This module also includes a [robots package](performance-poetry/complex-poetry/s
 that provides some utility helper 'robots' for the UiAutomator [androidx.test.uiautomator.UiDevice]
 as well as scenarios specific to this Complex Poetry application.
 
-### complex-benchmark
+#### complex-benchmark
 
 This is an Android Test module which hosts an application that can run androidx.macrobenchmarks.
 See the kdoc on [ComplexPoetryBenchmarks.](performance-poetry/complex-benchmark/src/main/java/com/squareup/benchmarks/performance/complex/poetry/benchmark/ComplexPoetryBenchmarks.kt)
 
 The results for this are stored in the same folder at [ComplexPoetryResults.txt.](performance-poetry/complex-benchmark/src/main/java/com/squareup/benchmarks/performance/complex/poetry/benchmark/ComplexPoetryResults.txt)
+
+## Micro benchmarks
+
+### runtime-microbenchmark
+
+This module contains benchmarks that measure the performance of the raw workflow runtime. To get
+accurate results, these benchmarks must be ran on real physical devices, not emulators.
+
+You can run them from the IDE like any other test (recommended), or with this command:
+```bash
+./gradlew :benchmarks:runtime-microbenchmark:connectedReleaseAndroidTest
+```
+
+We do not run benchmarks in CI, but we do validate that they compile by running them in "dry mode".

benchmarks/dungeon-benchmark/build.gradle.kts

@@ -48,7 +48,7 @@ kotlin {
 }
 
 dependencies {
-  implementation(libs.androidx.macro.benchmark)
+  implementation(libs.androidx.benchmark.macro)
   implementation(libs.androidx.test.espresso.core)
   implementation(libs.androidx.test.junit)
   implementation(libs.androidx.test.uiautomator)

benchmarks/performance-poetry/complex-benchmark/build.gradle.kts

@@ -57,7 +57,7 @@ kotlin {
 }
 
 dependencies {
-  implementation(libs.androidx.macro.benchmark)
+  implementation(libs.androidx.benchmark.macro)
   implementation(libs.androidx.test.espresso.core)
   implementation(libs.androidx.test.junit)
   implementation(libs.androidx.test.uiautomator)

benchmarks/runtime-microbenchmark/benchmark-proguard-rules.pro

@@ -0,0 +1,37 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile
+
+-dontobfuscate
+
+-ignorewarnings
+
+-keepattributes *Annotation*
+
+-dontnote junit.framework.**
+-dontnote junit.runner.**
+
+-dontwarn androidx.test.**
+-dontwarn org.junit.**
+-dontwarn org.hamcrest.**
+-dontwarn com.squareup.javawriter.JavaWriter
+
+-keepclasseswithmembers @org.junit.runner.RunWith public class *

benchmarks/runtime-microbenchmark/build.gradle.kts

@@ -0,0 +1,77 @@
+import com.rickbusarow.kgx.libsCatalog
+import com.rickbusarow.kgx.version
+import com.squareup.workflow1.buildsrc.internal.javaTarget
+import com.squareup.workflow1.buildsrc.internal.javaTargetVersion
+
+plugins {
+  // Must be applied before kotlin-android so the convention can detect that this is a benchmark.
+  alias(libs.plugins.androidx.benchmark)
+  id("com.android.library")
+  id("kotlin-android")
+  id("app.cash.burst")
+}
+
+// Note: We are not including our defaults from .buildscript as we do not need the base Workflow
+// dependencies that those include.
+
+android {
+  compileSdk = libsCatalog.version("compileSdk").toInt()
+
+  compileOptions {
+    sourceCompatibility = javaTargetVersion
+    targetCompatibility = javaTargetVersion
+  }
+
+  kotlinOptions {
+    jvmTarget = javaTarget
+    freeCompilerArgs += "-opt-in=kotlin.RequiresOptIn"
+  }
+
+  defaultConfig {
+    minSdk = 28
+    targetSdk = libsCatalog.version("targetSdk").toInt()
+
+    testInstrumentationRunner = "androidx.benchmark.junit4.AndroidBenchmarkRunner"
+
+    // This flag is supposed to enable dry run in the test runner, and it does so locally, but for
+    // some reason it doesn't seem to be working in CI.
+    val benchmarkDryRunEnabled = project.findProperty("androidx.benchmark.dryRunMode.enable")
+    if (benchmarkDryRunEnabled == "true") {
+      println("Running benchmarks in dry mode: emulator allowed, no measurements taken, no warmup.")
+      testInstrumentationRunnerArguments["androidx.benchmark.dryRunMode.enable"] = "true"
+    } else {
+      // must be one of: 'None', 'StackSampling', or 'MethodTracing'
+      testInstrumentationRunnerArguments["androidx.benchmark.profiling.mode"] = "MethodTracing"
+      testInstrumentationRunnerArguments["androidx.benchmark.output.enable"] = "true"
+    }
+  }
+
+  testBuildType = "release"
+  buildTypes {
+    debug {
+      // Since isDebuggable can"t be modified by gradle for library modules,
+      // it must be done in a manifest - see src/androidTest/AndroidManifest.xml
+      isMinifyEnabled = true
+      proguardFiles(
+        getDefaultProguardFile("proguard-android-optimize.txt"),
+        "benchmark-proguard-rules.pro"
+      )
+    }
+    release {
+      isDefault = true
+    }
+  }
+
+  namespace = "com.squareup.benchmark.runtime.benchmark"
+  testNamespace = "$namespace.test"
+}
+
+dependencies {
+  androidTestImplementation(project(":workflow-runtime"))
+  androidTestImplementation(libs.androidx.benchmark)
+  androidTestImplementation(libs.androidx.test.espresso.core)
+  androidTestImplementation(libs.androidx.test.junit)
+  androidTestImplementation(libs.androidx.test.uiautomator)
+  androidTestImplementation(libs.kotlin.test.jdk)
+  androidTestImplementation(libs.kotlinx.coroutines.test)
+}

benchmarks/runtime-microbenchmark/src/androidTest/AndroidManifest.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools">
+
+  <!--
+      Important: disable debugging for accurate performance results
+
+      In a com.android.library project, this flag must be disabled from this
+      manifest, as it is not possible to override this flag from Gradle.
+    -->
+  <application
+      android:debuggable="false"
+      tools:ignore="HardcodedDebugMode"
+      tools:replace="android:debuggable">
+    <profileable android:shell="true"/>
+  </application>
+</manifest>