feat: integrate CoreML icon detection (OmniParser icon_detect_v1_5)

internal/appuse/display.go

@@ -24,10 +24,11 @@ type Rect struct {
 }
 
 type DisplayElement struct {
-	IsAX  bool
-	Hint  string
-	Name  string
-	Frame Rect
+	IsAX   bool
+	IsIcon bool
+	Hint   string
+	Name   string
+	Frame  Rect
 	// AX specific
 	Role                string
 	Label               string
@@ -172,6 +173,31 @@ func PrintSnapshot(jsonOutput string, debug bool) []AreaNode {
 		displayElements = append(displayElements, de)
 	}
 
+	for _, icon := range resp.IconElements {
+		// Filter icon elements outside window bounds
+		if icon.Frame.X < wx-1 || icon.Frame.Y < wy-1 ||
+			icon.Frame.X+icon.Frame.Width > wx+ww+1 ||
+			icon.Frame.Y+icon.Frame.Height > wy+wh+1 {
+			continue
+		}
+
+		// Convert screen points to window-relative pixels
+		px := (icon.Frame.X - wx) * scaleX
+		py := (icon.Frame.Y - wy) * scaleY
+		pw := icon.Frame.Width * scaleX
+		ph := icon.Frame.Height * scaleY
+
+		de := DisplayElement{
+			IsAX:                false,
+			IsIcon:              true,
+			Hint:                icon.Hint,
+			Name:                fmt.Sprintf("Icon (%.0f%%)", icon.Confidence*100),
+			OriginalFingerprint: getIconFingerprint(icon),
+		}
+		de.Frame.X, de.Frame.Y, de.Frame.Width, de.Frame.Height = px, py, pw, ph
+		displayElements = append(displayElements, de)
+	}
+
 	var areas []AreaNode
 	if len(displayElements) > 0 {
 		areas = printAreaHierarchy(img, imgBounds, displayElements, wx, wy, ww, wh, scaleX, scaleY, prevMap, debug)
@@ -228,6 +254,10 @@ func getOCRFingerprint(ocr OCRElement) string {
 	return "OCR|" + ocr.Name
 }
 
+func getIconFingerprint(icon IconElement) string {
+	return fmt.Sprintf("ICON|%.0f,%.0f,%.0f,%.0f", icon.Frame.X, icon.Frame.Y, icon.Frame.Width, icon.Frame.Height)
+}
+
 type displayTreeNode struct {
 	e        DisplayElement
 	children []*displayTreeNode
@@ -309,7 +339,9 @@ func printAreaHierarchy(img image.Image, imgBounds image.Rectangle, elements []D
 	printElementNode = func(n *displayTreeNode, baseIndent, extraSpace string) {
 		e := n.e
 		ocrChar := " "
-		if !e.IsAX || strings.Contains(e.Name, " (via OCR)") {
+		if e.IsIcon {
+			ocrChar = "#"
+		} else if !e.IsAX || strings.Contains(e.Name, " (via OCR)") {
 			ocrChar = "*"
 		}
 		diffChar := " "
@@ -342,9 +374,12 @@ func printAreaHierarchy(img image.Image, imgBounds image.Rectangle, elements []D
 		// Draw bounding boxes for all elements
 		green := color.RGBA{0, 255, 0, 255}
 		yellow := color.RGBA{255, 255, 0, 255}
+		red := color.RGBA{255, 0, 0, 255}
 		for _, e := range elements {
 			c := green
-			if !e.IsAX {
+			if e.IsIcon {
+				c = red
+			} else if !e.IsAX {
 				c = yellow
 			}
 			drawRect(debugImg, e.Frame, c)

internal/appuse/models.go

@@ -54,10 +54,22 @@ type SnapshotResponse struct {
 	Elements         []ElementNode   `json:"elements"`
 	FrontmostWindow  *BreadcrumbNode `json:"frontmostWindow"`
 	OCRElements      []OCRElement    `json:"ocrElements,omitempty"`
+	IconElements     []IconElement   `json:"iconElements,omitempty"`
 	Areas            []AreaNode      `json:"areas,omitempty"`
 	Caret            *CaretRect      `json:"caret,omitempty"`
 }
 
+type IconElement struct {
+	Confidence float64 `json:"confidence"`
+	Hint       string  `json:"hint"`
+	Frame      struct {
+		X      float64 `json:"x"`
+		Y      float64 `json:"y"`
+		Width  float64 `json:"width"`
+		Height float64 `json:"height"`
+	} `json:"frame"`
+}
+
 type AreaNode struct {
 	Name  string `json:"name"`
 	Hint  string `json:"hint"`

internal/appuse/service.go

@@ -228,6 +228,13 @@ func FindHint(hint string, targetAppName string) (float64, float64, bool) {
 		}
 	}
 
+	// Search in Icon elements
+	for _, icon := range resp.IconElements {
+		if strings.ToUpper(icon.Hint) == hint {
+			return icon.Frame.X + icon.Frame.Width/2, icon.Frame.Y + icon.Frame.Height/2, true
+		}
+	}
+
 	return 0, 0, false
 }