Skip to content

Commit f8af85b

Browse files
steve-the-edwardssteve-the-edwards
authored
Merge pull request #867 from square/sedwards/runtime-prototypes
Remove FrameTimeout Runtime; Retry Espresso Tests.
2 parents 7cf92f3 + 0ebc57d commit f8af85b

20 files changed

Lines changed: 74 additions & 298 deletions

File tree

‎.github/workflows/kotlin.yml‎

Lines changed: 0 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -100,34 +100,6 @@ jobs :
100100
with :
101101
report_paths : '**/build/test-results/test/TEST-*.xml'
102102

103-
jvm-timeout-runtime-test:
104-
name: Frame Timeout Runtime JVM Tests
105-
runs-on : ubuntu-latest
106-
timeout-minutes : 20
107-
steps :
108-
- uses : actions/checkout@v3
109-
- uses : gradle/wrapper-validation-action@v1
110-
- name : set up JDK 11
111-
uses : actions/setup-java@v3
112-
with :
113-
distribution: 'zulu'
114-
java-version : 11
115-
116-
## Actual task
117-
- uses: gradle/gradle-build-action@v2
118-
name : Check with Gradle
119-
with :
120-
arguments : |
121-
jvmTest --stacktrace --continue -Pworkflow.runtime=timeout
122-
cache-read-only: false
123-
124-
# Report as Github Pull Request Check.
125-
- name : Publish Test Report
126-
uses : mikepenz/action-junit-report@v3
127-
if : always() # always run even if the previous step fails
128-
with :
129-
report_paths : '**/build/test-results/test/TEST-*.xml'
130-
131103
ios-tests :
132104
name : iOS Tests
133105
runs-on : macos-latest
@@ -256,55 +228,6 @@ jobs :
256228
name : instrumentation-test-results-${{ matrix.api-level }}
257229
path : ./**/build/reports/androidTests/connected/**
258230

259-
# Turned off due to #850 which re-uses RenderContext.
260-
# frame-timeout-instrumentation-tests :
261-
# name : Frame Timeout Instrumentation tests
262-
# runs-on : macos-latest
263-
# timeout-minutes : 45
264-
# strategy :
265-
# # Allow tests to continue on other devices if they fail on one device.
266-
# fail-fast : false
267-
# matrix :
268-
# api-level :
269-
# - 29
270-
# # Unclear that older versions actually honor command to disable animation.
271-
# # Newer versions are reputed to be too slow: https://github.com/ReactiveCircus/android-emulator-runner/issues/222
272-
# steps :
273-
# - uses : actions/checkout@v3
274-
# - name : set up JDK 11
275-
# uses : actions/setup-java@v3
276-
# with :
277-
# distribution : 'zulu'
278-
# java-version : 11
279-
280-
# ## Build before running tests, using cache.
281-
# - uses: gradle/gradle-build-action@v2
282-
# name : Build instrumented tests
283-
# with :
284-
# # Unfortunately I don't think we can key this cache based on our project property so
285-
# # we clean and rebuild.
286-
# arguments : |
287-
# clean assembleDebugAndroidTest --stacktrace -Pworkflow.runtime=timeout
288-
# cache-read-only: false
289-
290-
# ## Actual task
291-
# - name : Instrumentation Tests
292-
# uses : reactivecircus/android-emulator-runner@v2
293-
# with :
294-
# # @ychescale9 suspects Galaxy Nexus is the fastest one
295-
# profile : Galaxy Nexus
296-
# api-level : ${{ matrix.api-level }}
297-
# arch : x86_64
298-
# # Skip the benchmarks as this is running on emulators
299-
# script : ./gradlew connectedCheck -x :benchmarks:dungeon-benchmark:connectedCheck -x :benchmarks:performance-poetry:complex-benchmark:connectedCheck -x :benchmarks:performance-poetry:complex-poetry:connectedCheck --stacktrace -Pworkflow.runtime=timeout
300-
301-
# - name : Upload results
302-
# if : ${{ always() }}
303-
# uses : actions/upload-artifact@v3
304-
# with :
305-
# name : instrumentation-test-results-${{ matrix.api-level }}
306-
# path : ./**/build/reports/androidTests/connected/**
307-
308231
upload-to-mobiledev :
309232
name : mobile.dev | Build & Upload
310233
runs-on : ubuntu-latest

‎benchmarks/performance-poetry/complex-benchmark/src/main/java/com/squareup/benchmarks/performance/complex/poetry/benchmark/ComplexPoetryBenchmarks.kt‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -132,8 +132,8 @@ class ComplexPoetryBenchmarks {
132132
) {
133133
startActivityAndWait { intent ->
134134
intent.apply {
135-
putExtra(PerformancePoetryActivity.EXTRA_PERF_CONFIG_INITIALIZING, true)
136-
putExtra(PerformancePoetryActivity.EXTRA_PERF_CONFIG_RENDERING, true)
135+
putExtra(PerformancePoetryActivity.EXTRA_SCENARIO_CONFIG_INITIALIZING, true)
136+
putExtra(PerformancePoetryActivity.EXTRA_TRACE_RENDER_PASSES, true)
137137
if (useHighFrequencyEvents) {
138138
putExtra(
139139
PerformancePoetryActivity.EXTRA_PERF_CONFIG_REPEAT,
@@ -203,8 +203,8 @@ class ComplexPoetryBenchmarks {
203203
PerformancePoetryActivity.HIGH_FREQUENCY_REPEAT_COUNT
204204
)
205205
}
206-
putExtra(PerformancePoetryActivity.EXTRA_PERF_CONFIG_INITIALIZING, true)
207-
putExtra(PerformancePoetryActivity.EXTRA_PERF_CONFIG_ACTION_TRACING, true)
206+
putExtra(PerformancePoetryActivity.EXTRA_SCENARIO_CONFIG_INITIALIZING, true)
207+
putExtra(PerformancePoetryActivity.EXTRA_TRACE_ACTION_TRACING, true)
208208
putExtra(PerformancePoetryActivity.EXTRA_TRACE_SELECT_TIMEOUTS, true)
209209
}
210210
}
@@ -251,8 +251,8 @@ class ComplexPoetryBenchmarks {
251251
@Test fun benchmarkLatencyWithFrameCallbacks() {
252252
fun addLatencyTracing(intent: Intent) {
253253
intent.apply {
254-
putExtra(PerformancePoetryActivity.EXTRA_PERF_CONFIG_INITIALIZING, true)
255-
putExtra(PerformancePoetryActivity.EXTRA_PERF_CONFIG_FRAME_LATENCY, true)
254+
putExtra(PerformancePoetryActivity.EXTRA_SCENARIO_CONFIG_INITIALIZING, true)
255+
putExtra(PerformancePoetryActivity.EXTRA_TRACE_FRAME_LATENCY, true)
256256
}
257257
}
258258

‎benchmarks/performance-poetry/complex-poetry/src/androidTest/java/com/squareup/benchmarks/performance/complex/poetry/RenderPassTest.kt‎

Lines changed: 9 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,15 @@ import androidx.test.core.app.ApplicationProvider
66
import androidx.test.ext.junit.runners.AndroidJUnit4
77
import androidx.test.platform.app.InstrumentationRegistry
88
import androidx.test.uiautomator.UiDevice
9-
import com.squareup.benchmarks.performance.complex.poetry.PerformancePoetryActivity.Companion.EXTRA_PERF_CONFIG_INITIALIZING
109
import com.squareup.benchmarks.performance.complex.poetry.PerformancePoetryActivity.Companion.EXTRA_PERF_CONFIG_REPEAT
1110
import com.squareup.benchmarks.performance.complex.poetry.PerformancePoetryActivity.Companion.EXTRA_PERF_CONFIG_SIMULTANEOUS
12-
import com.squareup.benchmarks.performance.complex.poetry.PerformancePoetryActivity.Companion.EXTRA_RUNTIME_FRAME_TIMEOUT
11+
import com.squareup.benchmarks.performance.complex.poetry.PerformancePoetryActivity.Companion.EXTRA_SCENARIO_CONFIG_INITIALIZING
1312
import com.squareup.benchmarks.performance.complex.poetry.cyborgs.landscapeOrientation
1413
import com.squareup.benchmarks.performance.complex.poetry.cyborgs.openRavenAndNavigate
1514
import com.squareup.benchmarks.performance.complex.poetry.cyborgs.waitForPoetry
1615
import com.squareup.benchmarks.performance.complex.poetry.instrumentation.RenderPassCountingInterceptor
1716
import org.junit.Assert.fail
1817
import org.junit.Before
19-
import org.junit.Ignore
2018
import org.junit.Test
2119
import org.junit.runner.RunWith
2220

@@ -32,7 +30,6 @@ class RenderPassTest {
3230
val useHighFrequencyRange: Boolean,
3331
val simultaneousActions: Int,
3432
val baselineExpectation: RenderExpectation,
35-
val frameTimeoutExpectation: RenderExpectation
3633
)
3734

3835
data class RenderExpectation(
@@ -58,62 +55,35 @@ class RenderPassTest {
5855
}
5956

6057
@Test fun renderPassCounterBaselineComplexWithInitializingState() {
61-
runRenderPassCounter(COMPLEX_INITIALIZING, useFrameTimeout = false)
58+
runRenderPassCounter(COMPLEX_INITIALIZING)
6259
}
6360

64-
@Ignore
6561
@Test fun renderPassCounterBaselineComplexNoInitializingState() {
66-
runRenderPassCounter(COMPLEX_NO_INITIALIZING, useFrameTimeout = false)
62+
runRenderPassCounter(COMPLEX_NO_INITIALIZING)
6763
}
6864

69-
@Ignore
7065
@Test fun renderPassCounterBaselineComplexNoInitializingStateHighFrequencyEvents() {
71-
runRenderPassCounter(COMPLEX_NO_INITIALIZING_HIGH_FREQUENCY, useFrameTimeout = false)
66+
runRenderPassCounter(COMPLEX_NO_INITIALIZING_HIGH_FREQUENCY)
7267
}
7368

74-
@Ignore
7569
@Test fun renderPassCounterBaselineComplexNoInitializingStateSimultaneous() {
76-
runRenderPassCounter(COMPLEX_NO_INITIALIZING_SIMULTANEOUS, useFrameTimeout = false)
77-
}
78-
79-
@Ignore
80-
@Test fun renderPassCounterFrameTimeoutComplexWithInitializingState() {
81-
runRenderPassCounter(COMPLEX_INITIALIZING, useFrameTimeout = true)
82-
}
83-
84-
@Ignore
85-
@Test fun renderPassCounterFrameTimeoutComplexNoInitializingState() {
86-
runRenderPassCounter(COMPLEX_NO_INITIALIZING, useFrameTimeout = true)
87-
}
88-
89-
@Ignore
90-
@Test fun renderPassCounterFrameTimeoutComplexNoInitializingStateHighFrequencyEvents() {
91-
runRenderPassCounter(COMPLEX_NO_INITIALIZING_HIGH_FREQUENCY, useFrameTimeout = true)
92-
}
93-
94-
@Ignore
95-
@Test fun renderPassCounterFrameTimeoutComplexNoInitializingStateSimultaneous() {
96-
runRenderPassCounter(COMPLEX_NO_INITIALIZING_SIMULTANEOUS, useFrameTimeout = true)
70+
runRenderPassCounter(COMPLEX_NO_INITIALIZING_SIMULTANEOUS)
9771
}
9872

9973
private fun runRenderPassCounter(
10074
scenario: Scenario,
101-
useFrameTimeout: Boolean
10275
) {
10376
val intent = Intent(context, PerformancePoetryActivity::class.java).apply {
10477
addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
10578
addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK)
10679
putExtra(
107-
EXTRA_PERF_CONFIG_INITIALIZING,
80+
EXTRA_SCENARIO_CONFIG_INITIALIZING,
10881
scenario.useInitializingState
10982
)
11083
putExtra(
11184
EXTRA_PERF_CONFIG_SIMULTANEOUS,
11285
scenario.simultaneousActions
11386
)
114-
if (useFrameTimeout) {
115-
putExtra(EXTRA_RUNTIME_FRAME_TIMEOUT, useFrameTimeout)
116-
}
11787
if (scenario.useHighFrequencyRange) {
11888
putExtra(EXTRA_PERF_CONFIG_REPEAT, PerformancePoetryActivity.HIGH_FREQUENCY_REPEAT_COUNT)
11989
}
@@ -129,15 +99,9 @@ class RenderPassTest {
12999

130100
device.openRavenAndNavigate()
131101

132-
val expectation =
133-
if (useFrameTimeout) scenario.frameTimeoutExpectation else
134-
scenario.baselineExpectation
102+
val expectation = scenario.baselineExpectation
135103

136-
val title = if (useFrameTimeout) {
137-
"Runtime: FrameTimeout; "
138-
} else {
139-
"Runtime: RenderPerAction; "
140-
} + scenario.title
104+
val title = "Runtime: RenderPerAction; " + scenario.title
141105

142106
val totalRenderPasses = renderPassCountingInterceptor.renderEfficiencyTracking.totalRenderPasses
143107
val renderPassSubject = "the number of Render Passes (lower better)"
@@ -243,11 +207,6 @@ class RenderPassTest {
243207
freshRenderedNodes = 85..85,
244208
staleRenderedNodes = 608..608
245209
),
246-
frameTimeoutExpectation = RenderExpectation(
247-
totalPasses = 41..42,
248-
freshRenderedNodes = 85..85,
249-
staleRenderedNodes = 436..436
250-
)
251210
)
252211

253212
val COMPLEX_NO_INITIALIZING = Scenario(
@@ -256,31 +215,12 @@ class RenderPassTest {
256215
useHighFrequencyRange = false,
257216
simultaneousActions = 0,
258217
baselineExpectation = RenderExpectation(
259-
totalPasses = 54..54,
218+
totalPasses = 56..56,
260219
freshRenderedNodes = 83..83,
261220
staleRenderedNodes = 605..605
262221
),
263-
frameTimeoutExpectation = RenderExpectation(
264-
totalPasses = 40..41,
265-
freshRenderedNodes = 83..83,
266-
staleRenderedNodes = 431..431
267-
)
268222
)
269223

270-
/**
271-
* Note that for this scenario the frame timeout runtime can have different results
272-
* depending on what device/emulator it is run on.
273-
*
274-
* Physical Pixel 6:
275-
* ```
276-
RenderExpectation(
277-
totalPasses = 56..61,
278-
freshRenderedNodes = 106..108,
279-
staleRenderedNodes = 679..698
280-
)
281-
* ```
282-
* We use the values expected in CI for what we commit to the repo.
283-
*/
284224
val COMPLEX_NO_INITIALIZING_HIGH_FREQUENCY = Scenario(
285225
title = "the 'Raven navigation (no initializing state) scenario with high frequency events'",
286226
useInitializingState = false,
@@ -291,11 +231,6 @@ class RenderPassTest {
291231
freshRenderedNodes = 213..213,
292232
staleRenderedNodes = 2350..2350
293233
),
294-
frameTimeoutExpectation = RenderExpectation(
295-
totalPasses = 87..97, // On Pixel 6: 56..61
296-
freshRenderedNodes = 106..108,
297-
staleRenderedNodes = 679..698
298-
)
299234
)
300235

301236
val COMPLEX_NO_INITIALIZING_SIMULTANEOUS = Scenario(
@@ -309,11 +244,6 @@ class RenderPassTest {
309244
freshRenderedNodes = 253..253,
310245
staleRenderedNodes = 38919..38919
311246
),
312-
frameTimeoutExpectation = RenderExpectation(
313-
totalPasses = 88..99, // on Pixel 6: 56..61,
314-
freshRenderedNodes = 176..180,
315-
staleRenderedNodes = 4690..4700
316-
)
317247
)
318248

319249
fun congrats(

‎benchmarks/performance-poetry/complex-poetry/src/main/java/com/squareup/benchmarks/performance/complex/poetry/PerformancePoetryActivity.kt‎

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ import com.squareup.benchmarks.performance.complex.poetry.instrumentation.Simula
2121
import com.squareup.sample.container.SampleContainers
2222
import com.squareup.sample.poetry.model.Poem
2323
import com.squareup.workflow1.RuntimeConfig
24-
import com.squareup.workflow1.RuntimeConfig.FrameTimeout
2524
import com.squareup.workflow1.RuntimeConfig.RenderPerAction
2625
import com.squareup.workflow1.WorkflowExperimentalRuntime
2726
import com.squareup.workflow1.WorkflowInterceptor
@@ -64,12 +63,12 @@ class PerformancePoetryActivity : AppCompatActivity() {
6463
val simulatedPerfConfig = SimulatedPerfConfig(
6564
isComplex = true,
6665
complexityDelay = intent.getLongExtra(EXTRA_PERF_CONFIG_DELAY, 200L),
67-
useInitializingState = intent.getBooleanExtra(EXTRA_PERF_CONFIG_INITIALIZING, false),
66+
useInitializingState = intent.getBooleanExtra(EXTRA_SCENARIO_CONFIG_INITIALIZING, false),
6867
repeatOnNext = intent.getIntExtra(EXTRA_PERF_CONFIG_REPEAT, 0),
6968
simultaneousActions = intent.getIntExtra(EXTRA_PERF_CONFIG_SIMULTANEOUS, 0),
70-
traceFrameLatency = intent.getBooleanExtra(EXTRA_PERF_CONFIG_FRAME_LATENCY, false),
71-
traceEventLatency = intent.getBooleanExtra(EXTRA_PERF_CONFIG_ACTION_TRACING, false),
72-
traceRenderingPasses = intent.getBooleanExtra(EXTRA_PERF_CONFIG_RENDERING, false)
69+
traceFrameLatency = intent.getBooleanExtra(EXTRA_TRACE_FRAME_LATENCY, false),
70+
traceEventLatency = intent.getBooleanExtra(EXTRA_TRACE_ACTION_TRACING, false),
71+
traceRenderingPasses = intent.getBooleanExtra(EXTRA_TRACE_RENDER_PASSES, false)
7372
)
7473

7574
require(!(simulatedPerfConfig.traceFrameLatency && simulatedPerfConfig.traceRenderingPasses)) {
@@ -84,8 +83,7 @@ class PerformancePoetryActivity : AppCompatActivity() {
8483
installedInterceptor = ActionHandlingTracingInterceptor()
8584
}
8685

87-
val isFrameTimeout = intent.getBooleanExtra(EXTRA_RUNTIME_FRAME_TIMEOUT, false)
88-
val runtimeConfig = if (isFrameTimeout) FrameTimeout() else RenderPerAction
86+
val runtimeConfig = RenderPerAction
8987

9088
val component =
9189
PerformancePoetryComponent(installedInterceptor, simulatedPerfConfig, runtimeConfig)
@@ -234,17 +232,16 @@ class PerformancePoetryActivity : AppCompatActivity() {
234232
const val EXTRA_TRACE_SELECT_TIMEOUTS =
235233
"complex.poetry.performance.config.trace.select.timeouts"
236234
const val EXTRA_TRACE_ALL_MAIN_THREAD_MESSAGES = "complex.poetry.performance.config.trace.main"
237-
const val EXTRA_PERF_CONFIG_INITIALIZING = "complex.poetry.performance.config.use.initializing"
238-
const val EXTRA_PERF_CONFIG_ACTION_TRACING =
235+
const val EXTRA_SCENARIO_CONFIG_INITIALIZING =
236+
"complex.poetry.performance.config.use.initializing"
237+
const val EXTRA_TRACE_ACTION_TRACING =
239238
"complex.poetry.performance.config.track.action.tracing"
240-
const val EXTRA_PERF_CONFIG_FRAME_LATENCY =
239+
const val EXTRA_TRACE_FRAME_LATENCY =
241240
"complex.poetry.performance.config.track.frame.latency"
242-
const val EXTRA_PERF_CONFIG_RENDERING = "complex.poetry.performance.config.track.rendering"
241+
const val EXTRA_TRACE_RENDER_PASSES = "complex.poetry.performance.config.track.rendering"
243242
const val EXTRA_PERF_CONFIG_REPEAT = "complex.poetry.performance.config.repeat.amount"
244243
const val EXTRA_PERF_CONFIG_DELAY = "complex.poetry.performance.config.delay.length"
245244
const val EXTRA_PERF_CONFIG_SIMULTANEOUS = "complex.poetry.performance.config.simultaneous"
246-
const val EXTRA_RUNTIME_FRAME_TIMEOUT =
247-
"complex.poetry.performance.config.runtime.frametimeout"
248245

249246
const val SELECT_ON_TIMEOUT_LOG_NAME =
250247
"kotlinx.coroutines.selects.SelectBuilderImpl\$onTimeout\$\$inlined\$Runnable"

‎samples/containers/hello-back-button/src/androidTest/java/com/squareup/sample/hellobackbutton/HelloBackButtonEspressoTest.kt‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import com.squareup.workflow1.ui.internal.test.DetectLeaksAfterTestSuccess
1212
import com.squareup.workflow1.ui.internal.test.IdlingDispatcherRule
1313
import com.squareup.workflow1.ui.internal.test.actuallyPressBack
1414
import com.squareup.workflow1.ui.internal.test.inAnyView
15+
import com.squareup.workflow1.ui.internal.test.retryBlocking
1516
import org.junit.Rule
1617
import org.junit.Test
1718
import org.junit.rules.RuleChain
@@ -26,7 +27,7 @@ class HelloBackButtonEspressoTest {
2627
.around(scenarioRule)
2728
.around(IdlingDispatcherRule)
2829

29-
@Test fun wrappedTakesPrecedence() {
30+
@Test fun wrappedTakesPrecedence() = retryBlocking {
3031
inAnyView(withId(R.id.hello_message)).apply {
3132
check(matches(withText("Able")))
3233
perform(click())
@@ -40,7 +41,7 @@ class HelloBackButtonEspressoTest {
4041
}
4142
}
4243

43-
@Test fun outerHandlerAppliesIfWrappedHandlerIsNull() {
44+
@Test fun outerHandlerAppliesIfWrappedHandlerIsNull() = retryBlocking {
4445
inAnyView(withId(R.id.hello_message)).apply {
4546
actuallyPressBack()
4647
inAnyView(withText("Are you sure you want to do this thing?"))

0 commit comments

Comments
 (0)