Skip to content

Commit 73e66a1

Browse files
enaboappsOwenMcGirr
enaboapps
and
OwenMcGirr
authored
Use collection row hints for item scan rows (#2549)
- Add collection row hint providers for accessibility nodes - Split geometry rows only when collection item row metadata is reliable - Cover conservative collection row grouping fallback behavior 🤖 Auto-generated Co-authored-by: Owen McGirr <o.a.mcgirr@gmail.com>
1 parent 7ff60bb commit 73e66a1

5 files changed

Lines changed: 313 additions & 7 deletions

File tree

‎app/src/main/java/com/enaboapps/switchify/service/scanning/tree/CollectionRowGrouping.kt‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
package com.enaboapps.switchify.service.scanning.tree
2+
3+
import com.enaboapps.switchify.service.scanning.ScanNodeInterface
4+
5+
internal object CollectionRowGrouping {
6+
fun refineRows(
7+
geometryRows: List<List<ScanNodeInterface>>,
8+
minimumHintedNodes: Int = 2,
9+
minimumHintCoverage: Double = 0.75
10+
): List<List<ScanNodeInterface>> {
11+
require(minimumHintedNodes > 0)
12+
require(minimumHintCoverage > 0.0 && minimumHintCoverage <= 1.0)
13+
14+
return geometryRows.flatMap { row ->
15+
refineRow(
16+
row = row,
17+
minimumHintedNodes = minimumHintedNodes,
18+
minimumHintCoverage = minimumHintCoverage
19+
)
20+
}
21+
}
22+
23+
private fun refineRow(
24+
row: List<ScanNodeInterface>,
25+
minimumHintedNodes: Int,
26+
minimumHintCoverage: Double
27+
): List<List<ScanNodeInterface>> {
28+
if (row.isEmpty()) return listOf(row)
29+
30+
val hintedNodes = row.mapNotNull { node ->
31+
val hint = (node as? CollectionRowHintProvider)?.getCollectionRowHint()
32+
if (hint != null && hint.rowIndex >= 0) {
33+
HintedNode(node, hint)
34+
} else {
35+
null
36+
}
37+
}
38+
val hintCoverage = hintedNodes.size.toDouble() / row.size.toDouble()
39+
val distinctRowIndexes = hintedNodes.map { it.hint.rowIndex }.distinct()
40+
41+
if (
42+
hintedNodes.size < minimumHintedNodes ||
43+
hintCoverage < minimumHintCoverage ||
44+
distinctRowIndexes.size < 2
45+
) {
46+
return listOf(row)
47+
}
48+
49+
val grouped = hintedNodes
50+
.groupBy { it.hint.rowIndex }
51+
.mapValues { (_, nodes) -> nodes.map { it.node }.toMutableList() }
52+
.toMutableMap()
53+
54+
val visualRows = grouped.mapValues { (_, nodes) -> nodes.averageMidY() }
55+
val unhintedNodes = row.filterNot { node -> hintedNodes.any { it.node === node } }
56+
unhintedNodes.forEach { node ->
57+
val nearestRowIndex = visualRows.minByOrNull { (_, visualMidY) ->
58+
kotlin.math.abs(node.getMidY() - visualMidY)
59+
}?.key
60+
61+
if (nearestRowIndex == null) return listOf(row)
62+
grouped.getValue(nearestRowIndex).add(node)
63+
}
64+
65+
return grouped.entries
66+
.sortedWith(
67+
compareBy<Map.Entry<Int, MutableList<ScanNodeInterface>>> { (_, nodes) ->
68+
nodes.minOf { it.getTop() }
69+
}.thenBy { (rowIndex, _) -> rowIndex }
70+
)
71+
.map { (_, nodes) -> nodes.sortedBy { it.getLeft() } }
72+
}
73+
74+
private data class HintedNode(
75+
val node: ScanNodeInterface,
76+
val hint: CollectionRowHint
77+
)
78+
79+
private fun List<ScanNodeInterface>.averageMidY(): Double {
80+
return sumOf { it.getMidY() }.toDouble() / size.toDouble()
81+
}
82+
}

‎app/src/main/java/com/enaboapps/switchify/service/scanning/tree/CollectionRowHint.kt‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
package com.enaboapps.switchify.service.scanning.tree
2+
3+
data class CollectionRowHint(
4+
val rowIndex: Int,
5+
val rowSpan: Int,
6+
val columnIndex: Int?,
7+
val columnSpan: Int?
8+
)
9+
10+
interface CollectionRowHintProvider {
11+
fun getCollectionRowHint(): CollectionRowHint?
12+
}

‎app/src/main/java/com/enaboapps/switchify/service/scanning/tree/ScanTreeBuilder.kt‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,23 @@ class ScanTreeBuilder(
2525
* @return A list of ScanTreeItems representing the built tree.
2626
*/
2727
fun buildTree(nodes: List<ScanNodeInterface>, itemThreshold: Int = 40): List<ScanTreeItem> {
28-
val tree = mutableListOf<ScanTreeItem>()
29-
if (nodes.isEmpty()) return tree
28+
if (nodes.isEmpty()) return emptyList()
29+
30+
val geometryRows = buildGeometryRows(nodes, itemThreshold)
31+
val rows = if (scanSettings.isRowColumnScanEnabled()) {
32+
CollectionRowGrouping.refineRows(geometryRows)
33+
} else {
34+
geometryRows
35+
}
36+
37+
return rows.map { createScanTreeItem(it) }
38+
}
39+
40+
private fun buildGeometryRows(
41+
nodes: List<ScanNodeInterface>,
42+
itemThreshold: Int
43+
): List<List<ScanNodeInterface>> {
44+
val rows = mutableListOf<List<ScanNodeInterface>>()
3045

3146
val sortedNodes = nodes.sortedBy { it.getMidY() }
3247
var currentTreeItem = mutableListOf<ScanNodeInterface>(sortedNodes.first())
@@ -46,7 +61,7 @@ class ScanTreeBuilder(
4661
currentTreeItem.add(node)
4762
} else {
4863
if (currentTreeItem.isNotEmpty()) {
49-
tree.add(createScanTreeItem(currentTreeItem))
64+
rows.add(currentTreeItem)
5065
currentTreeItem = mutableListOf()
5166
}
5267
currentTreeItem.add(node)
@@ -55,10 +70,10 @@ class ScanTreeBuilder(
5570
}
5671

5772
if (currentTreeItem.isNotEmpty()) {
58-
tree.add(createScanTreeItem(currentTreeItem))
73+
rows.add(currentTreeItem)
5974
}
6075

61-
return tree
76+
return rows
6277
}
6378

6479
/**
@@ -103,4 +118,4 @@ class ScanTreeBuilder(
103118
ScanTreeItem(sorted, sorted[0].getTop(), false)
104119
}
105120
}
106-
}
121+
}

‎app/src/main/java/com/enaboapps/switchify/service/techniques/nodes/Node.kt‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ import com.enaboapps.switchify.service.gestures.GesturePoint
1111
import com.enaboapps.switchify.service.gestures.placement.FingerMode
1212
import com.enaboapps.switchify.service.menu.MenuItem
1313
import com.enaboapps.switchify.service.scanning.ScanNodeInterface
14+
import com.enaboapps.switchify.service.scanning.tree.CollectionRowHint
15+
import com.enaboapps.switchify.service.scanning.tree.CollectionRowHintProvider
1416
import com.enaboapps.switchify.service.selection.SelectionHandler
1517
import com.enaboapps.switchify.service.techniques.nodes.scanners.NodeScannerUI
1618
import com.enaboapps.switchify.service.techniques.pointscan.blocks.PointScanBlock
@@ -34,7 +36,7 @@ data class NodeScanSignature(
3436
*/
3537
class Node(
3638
private var onSelect: (() -> Unit?)? = null
37-
) : ScanNodeInterface {
39+
) : ScanNodeInterface, CollectionRowHintProvider {
3840
private var nodeInfo: AccessibilityNodeInfo? = null
3941
private var x: Int = 0
4042
private var y: Int = 0
@@ -255,6 +257,22 @@ class Node(
255257
return capabilities?.prefersAccessibilityClickForSelection ?: true
256258
}
257259

260+
override fun getCollectionRowHint(): CollectionRowHint? {
261+
val item = capabilities
262+
?.collectionMetadata
263+
?.collectionItem
264+
?: return null
265+
266+
if (item.rowIndex < 0) return null
267+
268+
return CollectionRowHint(
269+
rowIndex = item.rowIndex,
270+
rowSpan = item.rowSpan,
271+
columnIndex = item.columnIndex.takeIf { it >= 0 },
272+
columnSpan = item.columnSpan.takeIf { it >= 0 }
273+
)
274+
}
275+
258276
override fun getContentDescription(): String {
259277
return contentDescription
260278
}

‎app/src/test/java/com/enaboapps/switchify/service/scanning/tree/CollectionRowGroupingTest.kt‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
package com.enaboapps.switchify.service.scanning.tree
2+
3+
import com.enaboapps.switchify.service.scanning.ScanNodeInterface
4+
import org.junit.Assert.assertEquals
5+
import org.junit.Assert.assertSame
6+
import org.junit.Test
7+
8+
class CollectionRowGroupingTest {
9+
@Test
10+
fun keepsGeometryRowsWhenNoHints() {
11+
val row = listOf(node("a", left = 0, top = 0), node("b", left = 100, top = 0))
12+
13+
val rows = CollectionRowGrouping.refineRows(listOf(row))
14+
15+
assertEquals(1, rows.size)
16+
assertEquals(listOf("a", "b"), rows[0].ids())
17+
}
18+
19+
@Test
20+
fun keepsGeometryRowsWhenOnlyOneHintedNode() {
21+
val row = listOf(
22+
node("a", left = 0, top = 0, rowIndex = 1),
23+
node("b", left = 100, top = 20)
24+
)
25+
26+
val rows = CollectionRowGrouping.refineRows(listOf(row))
27+
28+
assertEquals(1, rows.size)
29+
assertEquals(listOf("a", "b"), rows[0].ids())
30+
}
31+
32+
@Test
33+
fun keepsGeometryRowsWhenHintCoverageTooLow() {
34+
val row = listOf(
35+
node("a", left = 0, top = 0, rowIndex = 1),
36+
node("b", left = 100, top = 0, rowIndex = 2),
37+
node("c", left = 200, top = 0),
38+
node("d", left = 300, top = 0)
39+
)
40+
41+
val rows = CollectionRowGrouping.refineRows(listOf(row))
42+
43+
assertEquals(1, rows.size)
44+
assertEquals(listOf("a", "b", "c", "d"), rows[0].ids())
45+
}
46+
47+
@Test
48+
fun keepsGeometryRowsWhenAllHintsUseSameRowIndex() {
49+
val row = listOf(
50+
node("a", left = 100, top = 0, rowIndex = 1),
51+
node("b", left = 0, top = 0, rowIndex = 1)
52+
)
53+
54+
val rows = CollectionRowGrouping.refineRows(listOf(row))
55+
56+
assertEquals(1, rows.size)
57+
assertEquals(listOf("a", "b"), rows[0].ids())
58+
}
59+
60+
@Test
61+
fun splitsGeometryRowWhenHintsExposeMultipleRows() {
62+
val row = listOf(
63+
node("a", left = 100, top = 0, rowIndex = 1),
64+
node("b", left = 0, top = 0, rowIndex = 1),
65+
node("c", left = 100, top = 40, rowIndex = 2),
66+
node("d", left = 0, top = 40, rowIndex = 2)
67+
)
68+
69+
val rows = CollectionRowGrouping.refineRows(listOf(row))
70+
71+
assertEquals(2, rows.size)
72+
assertEquals(listOf("b", "a"), rows[0].ids())
73+
assertEquals(listOf("d", "c"), rows[1].ids())
74+
}
75+
76+
@Test
77+
fun sortsRefinedRowsByVisualTop() {
78+
val row = listOf(
79+
node("visuallySecond", left = 0, top = 40, rowIndex = 1),
80+
node("visuallyFirst", left = 0, top = 0, rowIndex = 2)
81+
)
82+
83+
val rows = CollectionRowGrouping.refineRows(listOf(row))
84+
85+
assertEquals(2, rows.size)
86+
assertEquals(listOf("visuallyFirst"), rows[0].ids())
87+
assertEquals(listOf("visuallySecond"), rows[1].ids())
88+
}
89+
90+
@Test
91+
fun sortsNodesLeftToRightWithinRefinedRows() {
92+
val row = listOf(
93+
node("right", left = 200, top = 0, rowIndex = 1),
94+
node("left", left = 0, top = 0, rowIndex = 1),
95+
node("bottom", left = 0, top = 40, rowIndex = 2)
96+
)
97+
98+
val rows = CollectionRowGrouping.refineRows(listOf(row))
99+
100+
assertEquals(2, rows.size)
101+
assertEquals(listOf("left", "right"), rows[0].ids())
102+
}
103+
104+
@Test
105+
fun ignoresInvalidNegativeRowIndex() {
106+
val row = listOf(
107+
node("invalid", left = 0, top = 0, rowIndex = -1),
108+
node("valid", left = 100, top = 0, rowIndex = 2)
109+
)
110+
111+
val rows = CollectionRowGrouping.refineRows(listOf(row))
112+
113+
assertEquals(1, rows.size)
114+
assertEquals(listOf("invalid", "valid"), rows[0].ids())
115+
}
116+
117+
@Test
118+
fun doesNotMergeSeparateGeometryRows() {
119+
val firstRow = listOf(node("a", left = 0, top = 0, rowIndex = 1))
120+
val secondRow = listOf(node("b", left = 0, top = 40, rowIndex = 1))
121+
122+
val rows = CollectionRowGrouping.refineRows(listOf(firstRow, secondRow))
123+
124+
assertEquals(2, rows.size)
125+
assertSame(firstRow[0], rows[0][0])
126+
assertSame(secondRow[0], rows[1][0])
127+
}
128+
129+
private fun node(
130+
id: String,
131+
left: Int,
132+
top: Int,
133+
rowIndex: Int? = null
134+
): TestScanNode {
135+
return TestScanNode(
136+
id = id,
137+
nodeLeft = left,
138+
nodeTop = top,
139+
nodeWidth = 20,
140+
nodeHeight = 20,
141+
rowIndex = rowIndex
142+
)
143+
}
144+
145+
private fun List<ScanNodeInterface>.ids(): List<String> {
146+
return map { (it as TestScanNode).id }
147+
}
148+
149+
private data class TestScanNode(
150+
val id: String,
151+
val nodeLeft: Int,
152+
val nodeTop: Int,
153+
val nodeWidth: Int,
154+
val nodeHeight: Int,
155+
val rowIndex: Int?
156+
) : ScanNodeInterface, CollectionRowHintProvider {
157+
override fun getCollectionRowHint(): CollectionRowHint? {
158+
return rowIndex?.let {
159+
CollectionRowHint(
160+
rowIndex = it,
161+
rowSpan = 1,
162+
columnIndex = null,
163+
columnSpan = null
164+
)
165+
}
166+
}
167+
168+
override fun getLeft(): Int = nodeLeft
169+
override fun getTop(): Int = nodeTop
170+
override fun getMidX(): Int = nodeLeft + nodeWidth / 2
171+
override fun getMidY(): Int = nodeTop + nodeHeight / 2
172+
override fun getWidth(): Int = nodeWidth
173+
override fun getHeight(): Int = nodeHeight
174+
override fun getContentDescription(): String = id
175+
override fun highlight() = Unit
176+
override fun unhighlight() = Unit
177+
override fun select() = Unit
178+
}
179+
}

0 commit comments

Comments
 (0)