fix(tldraw): bound selection edge and rotate handle hit areas (tldraw#8926)

In order to stop edge resize handles from stealing hits near the corners
(tldraw#8840), this PR bounds the edge and rotate handle hit geometry in
`SelectionForegroundOverlayUtil`. Edge handles were modeled as unfilled
`Edge2d` lines, which the `OverlayManager` inflates by the global
`hitTestMargin` (8/zoom on each side). That made the edge hit band wider
than the corner box (≈6.75/zoom half-extent), so the edge won hit tests
right at the corner. Edges are now bounded filled `Polygon2d` rectangles
matching the pre-overlay dimensions (`targetSize` per side, ≈4.5/zoom),
restoring the corner-wins-near-corners behavior. The rotate handle
radius is also set to exactly half the corner box width.

### Change type

- [x] `bugfix`

### Test plan

1. Select a shape and move the pointer just outside a corner — the
corner resize cursor should win over the edge cursor.
2. Hover the middle of an edge — the edge resize handle should still be
reachable.
3. Hover a rotate corner — its circular hit area should be half the
corner box width.

- [x] Unit tests

### Release notes

- Fix selection edge resize handles overlapping corner handles, which
made corners hard to grab on small shapes.

### Code changes

| Section   | LOC change |
| --------- | ---------- |
| Core code | +15 / -14  |
| Tests     | +7 / -7    |

Closes tldraw#8840
Relates to tldraw#8839

Author: steveruizok

packages/tldraw/src/lib/overlays/SelectionForegroundOverlayUtil.ts

@@ -1,7 +1,6 @@
 import {
 	Box,
 	Circle2d,
-	Edge2d,
 	Geometry2d,
 	HALF_PI,
 	Mat,
@@ -245,10 +244,12 @@ export class SelectionForegroundOverlayUtil extends OverlayUtil<TLSelectionForeg
 		transform: Mat
 	): Geometry2d {
 		if (handle === 'top' || handle === 'bottom' || handle === 'left' || handle === 'right') {
-			const edge = this._getEdgeLocalPoints(handle, state.width, state.height)
-			return new Edge2d({
-				start: Mat.applyToPoint(transform, edge.start),
-				end: Mat.applyToPoint(transform, edge.end),
+			const rect = this._getEdgeLocalRect(handle, state)
+			return new Polygon2d({
+				points: this._localRectToPoints(rect.x, rect.y, rect.w, rect.h).map((p) =>
+					Mat.applyToPoint(transform, p)
+				),
+				isFilled: true,
 			})
 		}
 
@@ -269,7 +270,7 @@ export class SelectionForegroundOverlayUtil extends OverlayUtil<TLSelectionForeg
 	): Geometry2d {
 		const cornerSize = Math.max(state.targetSizeX, state.targetSizeY) * 1.5
 		const center = this._getRotateHandleLocalCenter(handle, state.width, state.height, cornerSize)
-		const radius = (state.targetSize * 3) / 2
+		const radius = cornerSize
 		return new Circle2d({
 			x: center.x - radius,
 			y: center.y - radius,
@@ -622,20 +623,20 @@ export class SelectionForegroundOverlayUtil extends OverlayUtil<TLSelectionForeg
 		}
 	}
 
-	private _getEdgeLocalPoints(
+	private _getEdgeLocalRect(
 		edge: SelectionEdge,
-		width: number,
-		height: number
-	): { start: Vec; end: Vec } {
+		state: SelectionState
+	): { x: number; y: number; w: number; h: number } {
+		const { width, height, targetSizeX, targetSizeY } = state
 		switch (edge) {
 			case 'top':
-				return { start: new Vec(0, 0), end: new Vec(width, 0) }
+				return { x: 0, y: -targetSizeY, w: width, h: targetSizeY * 2 }
 			case 'right':
-				return { start: new Vec(width, 0), end: new Vec(width, height) }
+				return { x: width - targetSizeX, y: 0, w: targetSizeX * 2, h: height }
 			case 'bottom':
-				return { start: new Vec(0, height), end: new Vec(width, height) }
+				return { x: 0, y: height - targetSizeY, w: width, h: targetSizeY * 2 }
 			case 'left':
-				return { start: new Vec(0, 0), end: new Vec(0, height) }
+				return { x: -targetSizeX, y: 0, w: targetSizeX * 2, h: height }
 		}
 	}
 

packages/tldraw/src/test/overlays/OverlayManager.test.ts

@@ -98,16 +98,16 @@ describe('OverlayManager', () => {
 			expect(miss).toBeNull()
 		})
 
-		it('respects margin parameter for unfilled geometries (edge handles)', () => {
+		it('hits the bounded edge handle hit area regardless of margin', () => {
 			editor.createShapes([{ id: ids.box1, type: 'geo', x: 0, y: 0, props: { w: 100, h: 100 } }])
 			editor.select(ids.box1)
-			// Point near the top edge but slightly outside
-			const nearTop = { x: 50, y: -5 }
-			// With zero margin, should miss
-			expect(editor.overlays.getOverlayAtPoint(nearTop, 0)).toBeNull()
-			// With margin, should hit the top edge overlay
-			const hit = editor.overlays.getOverlayAtPoint(nearTop, 10)
+			// Edge handles are filled polygons of half-thickness targetSizeY (~4.5px at zoom 1);
+			// a point inside that band hits with margin 0, and a point outside misses.
+			const insideTopEdge = { x: 50, y: -3 }
+			const outsideTopEdge = { x: 50, y: -20 }
+			const hit = editor.overlays.getOverlayAtPoint(insideTopEdge, 0)
 			expect(hit?.id).toBe('selection_fg:top')
+			expect(editor.overlays.getOverlayAtPoint(outsideTopEdge, 0)).toBeNull()
 		})
 
 		it('returns first matching overlay when multiple overlap (corner)', () => {