Skip to content

Commit e63f240

Browse files
zach-klippensteinzach-klippenstein
committed
Added some basic microbenchmarks for the workflow runtime.
These will be expanded on to measure the Compose runtime implementation in #1442.
1 parent a013f39 commit e63f240

14 files changed

Lines changed: 373 additions & 21 deletions

File tree

‎.github/workflows/kotlin.yml‎

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -610,8 +610,8 @@ jobs:
610610
report_paths: '**/build/test-results/*[tT]est/TEST-*.xml'
611611
annotate_only: true # no write check perms on this workflow right now.
612612

613-
performance-tests:
614-
name: Performance tests
613+
macrobenchmarks:
614+
name: Macrobenchmarks
615615
runs-on: workflow-kotlin-test-runner-ubuntu-4core
616616
timeout-minutes: 45
617617
strategy:
@@ -635,6 +635,31 @@ jobs:
635635
failure-path-upload: '**/build/reports/androidTests/connected'
636636
failure-upload-name: 'perf-tests-report'
637637

638+
microbenchmarks:
639+
name: Microbenchmarks
640+
runs-on: workflow-kotlin-test-runner-ubuntu-4core
641+
timeout-minutes: 45
642+
strategy:
643+
# Allow tests to continue on other devices if they fail on one device.
644+
fail-fast: false
645+
matrix:
646+
api-level:
647+
- 31
648+
steps:
649+
- name: Checkout
650+
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
651+
652+
- name: Instrumented tests
653+
uses: ./.github/actions/gradle-tasks-with-emulator
654+
with:
655+
tests-name: perf-tests-results
656+
api-level: ${{ matrix.api-level }}
657+
prepare-task: :benchmarks:performance-poetry:complex-poetry:prepareDebugAndroidTestArtifacts
658+
# Run in dry run mode so they only run once, without warmup, since we don't care about measurements.
659+
# See https://developer.android.com/topic/performance/benchmarking/microbenchmark-instrumentation-args#dryrunmode-enable
660+
test-task: :benchmarks:runtime-microbenchmark:connectedReleaseAndroidTest -Pandroidx.benchmark.dryRunMode.enable=true --continue
661+
restore-cache-key: androidTest-build-artifacts
662+
638663
instrumentation-tests:
639664
name: Instrumentation tests
640665
runs-on: workflow-kotlin-test-runner-ubuntu-4core
@@ -659,7 +684,7 @@ jobs:
659684
api-level: ${{ matrix.api-level }}
660685
prepare-task: prepareConnectedCheckShard${{matrix.shardNum}}
661686
# -x excludes tasks
662-
test-task: connectedCheckShard${{matrix.shardNum}} -x :benchmarks:dungeon-benchmark:connectedAndroidTest -x :benchmarks:performance-poetry:complex-benchmark:connectedAndroidTest -x :benchmarks:performance-poetry:complex-poetry:connectedAndroidTest
687+
test-task: connectedCheckShard${{matrix.shardNum}} -x :benchmarks:dungeon-benchmark:connectedAndroidTest -x :benchmarks:performance-poetry:complex-benchmark:connectedAndroidTest -x :benchmarks:performance-poetry:complex-poetry:connectedAndroidTest -x :benchmarks:runtime-microbenchmark:connectedAndroidTest
663688
write-cache-key: androidTest-build-artifacts-${{matrix.shardNum}}
664689
restore-cache-key: main-build-artifacts
665690
failure-path-upload: '**/build/reports/androidTests/connected'
@@ -690,7 +715,7 @@ jobs:
690715
api-level: ${{ matrix.api-level }}
691716
prepare-task: prepareConnectedCheckShard${{matrix.shardNum}} -Pworkflow.runtime=${{matrix.runtime}}
692717
# -x excludes tasks
693-
test-task: connectedCheckShard${{matrix.shardNum}} -Pworkflow.runtime=${{matrix.runtime}} -x :benchmarks:dungeon-benchmark:connectedAndroidTest -x :benchmarks:performance-poetry:complex-benchmark:connectedAndroidTest -x :benchmarks:performance-poetry:complex-poetry:connectedAndroidTest
718+
test-task: connectedCheckShard${{matrix.shardNum}} -Pworkflow.runtime=${{matrix.runtime}} -x :benchmarks:dungeon-benchmark:connectedAndroidTest -x :benchmarks:performance-poetry:complex-benchmark:connectedAndroidTest -x :benchmarks:performance-poetry:complex-poetry:connectedAndroidTest -x :benchmarks:runtime-microbenchmark:connectedAndroidTest
694719
write-cache-key: androidTest-build-artifacts-${{matrix.shardNum}}-${{matrix.runtime}}
695720
restore-cache-key: main-build-artifacts
696721
failure-path-upload: '**/build/reports/androidTests/connected'
@@ -724,7 +749,8 @@ jobs:
724749
- unit-conflate-partial-drainExclusive-runtime-tests
725750
- unit-all-runtime-tests
726751
- ktlint
727-
- performance-tests
752+
- macrobenchmarks
753+
- microbenchmarks
728754
- runtime-instrumentation-tests
729755
- shards-and-version
730756
- tutorials

‎benchmarks/README.md‎

Lines changed: 30 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,18 @@ deterministic tests - such as those for render passes - can be run on any device
55
otherwise), but the benchmarks should be run on physical devices, or the closest approximation
66
to physical devices you can get.
77

8-
## Baseline Profiles
8+
There are two types of benchmarks:
9+
10+
- **Macro benchmarks** measure a particular flow in an entire app and are used to generate baseline
11+
profiles. They're analogous to integration tests.
12+
- **Micro benchmarks** measure a specific operation or set of small operations but running them
13+
many times and collecting aggregate statistics. They're used to compare detailed performance of
14+
competing implementations, detect specific performance regressions, and inform optimizations.
15+
They're analogous to unit tests.
16+
17+
## Macro benchmarks
18+
19+
### Baseline Profiles
920

1021
The sample apps can be used to extract 'baseline profiles' to improve code loading time after first
1122
install. See [baseline profiles](https://developer.android.com/studio/profile/baselineprofiles).
@@ -36,7 +47,7 @@ This will create an output file separated by module and then also by package as
3647
profile for each module can be added into its /src/main directory as `baseline-prof.txt`. Then on a
3748
release build this will be included with the resulting APK/binary.
3849

39-
## dungeon-benchmark
50+
### dungeon-benchmark
4051

4152
These are benchmarks for the [../samples/dungeon] app. Please instead use performance-poetry where
4253
possible.
@@ -57,11 +68,11 @@ same scenario with and without forcing the use of the profiles. To force the use
5768
`libs.androidx.profileinstaller` dependency is included into the app under profile (dungeon in this
5869
case) for side-loading the profiles.
5970

60-
## performance-poetry
71+
### performance-poetry
6172

6273
Module of code for performance testing related to poetry applications.
6374

64-
### complex-poetry
75+
#### complex-poetry
6576

6677
This application is a modified version of the samples/containers/app-poetry app which also uses the
6778
common components in samples/containers/common and samples/containers/poetry. It modifies this
@@ -92,9 +103,23 @@ This module also includes a [robots package](performance-poetry/complex-poetry/s
92103
that provides some utility helper 'robots' for the UiAutomator [androidx.test.uiautomator.UiDevice]
93104
as well as scenarios specific to this Complex Poetry application.
94105

95-
### complex-benchmark
106+
#### complex-benchmark
96107

97108
This is an Android Test module which hosts an application that can run androidx.macrobenchmarks.
98109
See the kdoc on [ComplexPoetryBenchmarks.](performance-poetry/complex-benchmark/src/main/java/com/squareup/benchmarks/performance/complex/poetry/benchmark/ComplexPoetryBenchmarks.kt)
99110

100111
The results for this are stored in the same folder at [ComplexPoetryResults.txt.](performance-poetry/complex-benchmark/src/main/java/com/squareup/benchmarks/performance/complex/poetry/benchmark/ComplexPoetryResults.txt)
112+
113+
## Micro benchmarks
114+
115+
### runtime-microbenchmark
116+
117+
This module contains benchmarks that measure the performance of the raw workflow runtime. To get
118+
accurate results, these benchmarks must be ran on real physical devices, not emulators.
119+
120+
You can run them from the IDE like any other test (recommended), or with this command:
121+
```bash
122+
./gradlew :benchmarks:runtime-microbenchmark:connectedReleaseAndroidTest
123+
```
124+
125+
We do not run benchmarks in CI, but we do validate that they compile by running them in "dry mode".

‎benchmarks/dungeon-benchmark/build.gradle.kts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ kotlin {
4848
}
4949

5050
dependencies {
51-
implementation(libs.androidx.macro.benchmark)
51+
implementation(libs.androidx.benchmark.macro)
5252
implementation(libs.androidx.test.espresso.core)
5353
implementation(libs.androidx.test.junit)
5454
implementation(libs.androidx.test.uiautomator)

‎benchmarks/performance-poetry/complex-benchmark/build.gradle.kts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ kotlin {
5757
}
5858

5959
dependencies {
60-
implementation(libs.androidx.macro.benchmark)
60+
implementation(libs.androidx.benchmark.macro)
6161
implementation(libs.androidx.test.espresso.core)
6262
implementation(libs.androidx.test.junit)
6363
implementation(libs.androidx.test.uiautomator)

‎benchmarks/runtime-microbenchmark/benchmark-proguard-rules.pro‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# Add project specific ProGuard rules here.
2+
# You can control the set of applied configuration files using the
3+
# proguardFiles setting in build.gradle.
4+
#
5+
# For more details, see
6+
# http://developer.android.com/guide/developing/tools/proguard.html
7+
8+
# If your project uses WebView with JS, uncomment the following
9+
# and specify the fully qualified class name to the JavaScript interface
10+
# class:
11+
#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
12+
# public *;
13+
#}
14+
15+
# Uncomment this to preserve the line number information for
16+
# debugging stack traces.
17+
#-keepattributes SourceFile,LineNumberTable
18+
19+
# If you keep the line number information, uncomment this to
20+
# hide the original source file name.
21+
#-renamesourcefileattribute SourceFile
22+
23+
-dontobfuscate
24+
25+
-ignorewarnings
26+
27+
-keepattributes *Annotation*
28+
29+
-dontnote junit.framework.**
30+
-dontnote junit.runner.**
31+
32+
-dontwarn androidx.test.**
33+
-dontwarn org.junit.**
34+
-dontwarn org.hamcrest.**
35+
-dontwarn com.squareup.javawriter.JavaWriter
36+
37+
-keepclasseswithmembers @org.junit.runner.RunWith public class *

‎benchmarks/runtime-microbenchmark/build.gradle.kts‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import com.rickbusarow.kgx.libsCatalog
2+
import com.rickbusarow.kgx.version
3+
import com.squareup.workflow1.buildsrc.internal.javaTarget
4+
import com.squareup.workflow1.buildsrc.internal.javaTargetVersion
5+
6+
plugins {
7+
id("com.android.library")
8+
id("kotlin-android")
9+
id("app.cash.burst")
10+
alias(libs.plugins.androidx.benchmark)
11+
}
12+
13+
// Note: We are not including our defaults from .buildscript as we do not need the base Workflow
14+
// dependencies that those include.
15+
16+
android {
17+
compileSdk = libsCatalog.version("compileSdk").toInt()
18+
19+
compileOptions {
20+
sourceCompatibility = javaTargetVersion
21+
targetCompatibility = javaTargetVersion
22+
}
23+
24+
kotlinOptions {
25+
jvmTarget = javaTarget
26+
freeCompilerArgs += "-opt-in=kotlin.RequiresOptIn"
27+
}
28+
29+
defaultConfig {
30+
minSdk = 28
31+
targetSdk = libsCatalog.version("targetSdk").toInt()
32+
33+
testInstrumentationRunner = "androidx.benchmark.junit4.AndroidBenchmarkRunner"
34+
35+
// must be one of: 'None', 'StackSampling', or 'MethodTracing'
36+
testInstrumentationRunnerArguments["androidx.benchmark.profiling.mode"] = "MethodTracing"
37+
testInstrumentationRunnerArguments["androidx.benchmark.output.enable"] = "true"
38+
}
39+
40+
testBuildType = "release"
41+
buildTypes {
42+
debug {
43+
// Since isDebuggable can"t be modified by gradle for library modules,
44+
// it must be done in a manifest - see src/androidTest/AndroidManifest.xml
45+
isMinifyEnabled = true
46+
proguardFiles(
47+
getDefaultProguardFile("proguard-android-optimize.txt"),
48+
"benchmark-proguard-rules.pro"
49+
)
50+
}
51+
release {
52+
isDefault = true
53+
}
54+
}
55+
56+
namespace = "com.squareup.benchmark.runtime.benchmark"
57+
testNamespace = "$namespace.test"
58+
}
59+
60+
dependencies {
61+
androidTestImplementation(project(":workflow-runtime"))
62+
androidTestImplementation(libs.androidx.benchmark)
63+
androidTestImplementation(libs.androidx.test.espresso.core)
64+
androidTestImplementation(libs.androidx.test.junit)
65+
androidTestImplementation(libs.androidx.test.uiautomator)
66+
androidTestImplementation(libs.kotlin.test.jdk)
67+
androidTestImplementation(libs.kotlinx.coroutines.test)
68+
}

‎benchmarks/runtime-microbenchmark/src/androidTest/AndroidManifest.xml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
3+
xmlns:tools="http://schemas.android.com/tools">
4+
5+
<!--
6+
Important: disable debugging for accurate performance results
7+
8+
In a com.android.library project, this flag must be disabled from this
9+
manifest, as it is not possible to override this flag from Gradle.
10+
-->
11+
<application
12+
android:debuggable="false"
13+
tools:ignore="HardcodedDebugMode"
14+
tools:replace="android:debuggable">
15+
<profileable android:shell="true"/>
16+
</application>
17+
</manifest>

0 commit comments

Comments
 (0)