Skip to content

Commit 44a50fc

Browse files
tanushahande2003tanushah
tanushahande2003
and
tanushah
authored
Improve mobile element highlighting and rectangle accuracy (#4480)
Enhances reliability of UI element highlighting and rectangle drawing, especially for Android automation. Ensures up-to-date element details are used, validates and refreshes element bounds before drawing overlays, and preserves correct element identity for web highlights. Adds error handling and makes driver highlight requests more robust, addressing issues with invalid rectangles and incorrect element selection in live spy and mobile scenarios. Co-authored-by: tanushah <tanushah@amdocs.com>
1 parent a9a2987 commit 44a50fc

4 files changed

Lines changed: 77 additions & 28 deletions

File tree

‎Ginger/Ginger/ApplicationModelsLib/POMModels/PomAllElementsPage.xaml.cs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -275,7 +275,12 @@ public void FocusSpyItemOnElementsGrid()
275275

276276
if (matchingOriginalElement == null)
277277
{
278-
mWinExplorer.LearnElementInfoDetails(mSpyElement);
278+
var learnedElement = mWinExplorer.LearnElementInfoDetails(mSpyElement);
279+
if (learnedElement != null)
280+
{
281+
mSpyElement = learnedElement;
282+
mSpyElement.WindowExplorer = mWinExplorer;
283+
}
279284
matchingOriginalElement = mWinExplorer.GetMatchingElement(mSpyElement, mPOM.GetUnifiedElementsList());
280285
}
281286

‎Ginger/Ginger/AutomatePageLib/AddActionMenu/LiveSpy/LiveSpyPage.xaml.cs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -265,8 +265,13 @@ private void SpyTimerHandler(object sender, EventArgs e)
265265
mWindowExplorerDriver.UnHighLightElements();
266266
mSpyElement.WindowExplorer = mWindowExplorerDriver;
267267

268-
// Learn details (fills properties/locators)
269-
xWindowSelectionUC.mWindowExplorerDriver.LearnElementInfoDetails(mSpyElement);
268+
// Learn details (fills properties/locators) and keep enriched instance if returned
269+
var learnedElement = xWindowSelectionUC.mWindowExplorerDriver.LearnElementInfoDetails(mSpyElement);
270+
if (learnedElement != null)
271+
{
272+
mSpyElement = learnedElement;
273+
mSpyElement.WindowExplorer = mWindowExplorerDriver;
274+
}
270275

271276
// Ensure element location/size fields are updated (important for mobile highlighting)
272277
try

‎Ginger/Ginger/AutomatePageLib/AddActionMenu/WindowExplorer/Common/WindowExplorerPage.xaml.cs‎

Lines changed: 43 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ limitations under the License.
2020
using Amdocs.Ginger.Common.Repository.ApplicationModelLib.POMModelLib;
2121
using Amdocs.Ginger.Common.UIElement;
2222
using Amdocs.Ginger.CoreNET;
23+
using Amdocs.Ginger.CoreNET.Drivers.CoreDrivers.Mobile;
2324
using Ginger.Actions.Locators.ASCF;
2425
using Ginger.Actions.UserControls;
2526
using Ginger.BusinessFlowsLibNew.AddActionMenu;
@@ -1441,7 +1442,7 @@ private async Task HighlightSelectedElement(System.Drawing.Point ClickedPoint)
14411442
}
14421443

14431444
// Draw rectangle in screenshot view using the element's (now fresh) bounds
1444-
DrawElementRectangleAsync(inspectElementInfo);
1445+
DrawElementRectangleAsync(inspectElementInfo, highlightInApp: false);
14451446

14461447
// Keep persistent highlighted element reference for UI flows
14471448
currentHighlightedElement = inspectElementInfo;
@@ -1482,13 +1483,33 @@ private async Task HighlightSelectedElement(System.Drawing.Point ClickedPoint)
14821483
}
14831484
}
14841485

1485-
private void DrawElementRectangleAsync(ElementInfo clickedElementInfo)
1486+
private void DrawElementRectangleAsync(ElementInfo clickedElementInfo, bool highlightInApp = true)
14861487
{
14871488
try
14881489
{
14891490
// remove previous rectangle
14901491
RemoveElemntRectangle();
14911492

1493+
// For some Android paths the element is identified but bounds are not populated yet.
1494+
// Refresh once before rectangle math; if still invalid, skip drawing a misleading rectangle.
1495+
if (clickedElementInfo.Width <= 0 || clickedElementInfo.Height <= 0)
1496+
{
1497+
try
1498+
{
1499+
mWindowExplorerDriver?.UpdateElementInfoFields(clickedElementInfo);
1500+
}
1501+
catch (Exception exUpd)
1502+
{
1503+
Reporter.ToLog(eLogLevel.DEBUG, "DrawElementRectangleAsync: UpdateElementInfoFields failed before rectangle draw: " + exUpd.Message, exUpd);
1504+
}
1505+
1506+
if (clickedElementInfo.Width <= 0 || clickedElementInfo.Height <= 0)
1507+
{
1508+
Reporter.ToLog(eLogLevel.DEBUG, "DrawElementRectangleAsync: skipping rectangle draw due to invalid element bounds.");
1509+
return;
1510+
}
1511+
}
1512+
14921513
// element bounds in source image/device coordinates
14931514
System.Drawing.Point ElementStartPoint = new System.Drawing.Point(clickedElementInfo.X, clickedElementInfo.Y);
14941515
System.Drawing.Point ElementMaxPoint = new System.Drawing.Point(clickedElementInfo.X + clickedElementInfo.Width, clickedElementInfo.Y + clickedElementInfo.Height);
@@ -1597,28 +1618,31 @@ private void DrawElementRectangleAsync(ElementInfo clickedElementInfo)
15971618
mScreenShotViewPage.xHighlighterBorder.Visibility = Visibility.Visible;
15981619
}), System.Windows.Threading.DispatcherPriority.Render);
15991620

1600-
// ask driver to highlight in the actual app as well (non-blocking)
1601-
try
1621+
if (highlightInApp)
16021622
{
1603-
var drv = mWindowExplorerDriver;
1604-
if (drv != null)
1623+
// ask driver to highlight in the actual app as well (non-blocking)
1624+
try
16051625
{
1606-
System.Threading.Tasks.Task.Run(() =>
1626+
var drv = mWindowExplorerDriver;
1627+
if (drv != null)
16071628
{
1608-
try
1609-
{
1610-
drv.HighLightElement(clickedElementInfo, locateElementByItLocators: true);
1611-
}
1612-
catch (Exception ex)
1629+
System.Threading.Tasks.Task.Run(() =>
16131630
{
1614-
Reporter.ToLog(eLogLevel.DEBUG, "Driver HighLightElement failed", ex);
1615-
}
1616-
});
1631+
try
1632+
{
1633+
drv.HighLightElement(clickedElementInfo, locateElementByItLocators: true);
1634+
}
1635+
catch (Exception ex)
1636+
{
1637+
Reporter.ToLog(eLogLevel.DEBUG, "Driver HighLightElement failed", ex);
1638+
}
1639+
});
1640+
}
1641+
}
1642+
catch (Exception ex)
1643+
{
1644+
Reporter.ToLog(eLogLevel.DEBUG, "Request to driver highlight element failed", ex);
16171645
}
1618-
}
1619-
catch (Exception ex)
1620-
{
1621-
Reporter.ToLog(eLogLevel.DEBUG, "Request to driver highlight element failed", ex);
16221646
}
16231647
}
16241648
catch (Exception ex)

‎Ginger/GingerCoreNET/Drivers/CoreDrivers/Mobile/Appium/GenericAppiumDriver.cs‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1794,20 +1794,24 @@ private void EnsureDeviceBounds(ElementInfo ei)
17941794
{
17951795
if (ei == null) return;
17961796

1797-
// If already have bounds property or width/height set - assume ok
1798-
if ((ei.Properties != null && ei.Properties.Any(p => string.Equals(p.Name, "bounds", StringComparison.InvariantCultureIgnoreCase)))
1799-
|| (ei.Width > 0 && ei.Height > 0))
1797+
// If we already have concrete size, nothing to do.
1798+
if (ei.Width > 0 && ei.Height > 0)
18001799
{
18011800
return;
18021801
}
18031802

18041803
// Try attribute "bounds" (common for Android UiAutomator)
1805-
string boundsAttr = null;
1804+
string boundsAttr = ei.Properties?
1805+
.FirstOrDefault(p => string.Equals(p.Name, "bounds", StringComparison.InvariantCultureIgnoreCase))
1806+
?.Value;
18061807
if (ei.ElementObject is IWebElement we)
18071808
{
18081809
try
18091810
{
1810-
boundsAttr = we.GetAttribute("bounds");
1811+
if (string.IsNullOrEmpty(boundsAttr))
1812+
{
1813+
boundsAttr = we.GetAttribute("bounds");
1814+
}
18111815
}
18121816
catch { boundsAttr = null; }
18131817
}
@@ -2932,6 +2936,16 @@ async void IWindowExplorer.HighLightElement(
29322936
ElementInfo filteredElementInfo =
29332937
POMExecutionUtils.FilterElementDetailsByCategory(ElementInfo, PomCategory);
29342938

2939+
// Keep exact spied element identity when caller did not request locator-based re-resolution.
2940+
// FilterElementDetailsByCategory may strip ElementObject in some flows, which can cause
2941+
// fallback XY lookup to highlight a nearby element (seen on iOS).
2942+
if (!locateElementByItLocators &&
2943+
filteredElementInfo.ElementObject == null &&
2944+
ElementInfo.ElementObject != null)
2945+
{
2946+
filteredElementInfo.ElementObject = ElementInfo.ElementObject;
2947+
}
2948+
29352949
((IWindowExplorer)mSeleniumDriver)
29362950
.HighLightElement(filteredElementInfo, locateElementByItLocators);
29372951

@@ -3088,7 +3102,6 @@ private ElementInfo GetElementAtMousePosition()
30883102
// ANDROID (mobile) + iOS → KEEP EXISTING LOGIC
30893103
// ================================================
30903104
foundNode = FindElementXmlNodeByXY(mousePosCurrent.X, mousePosCurrent.Y, false).Result;
3091-
30923105
if (foundNode != null)
30933106
{
30943107
foundElement = GetElementInfoforXmlNode(foundNode).Result;
@@ -3398,6 +3411,8 @@ private async Task<ElementInfo> GetElementInfoforXmlNode(XmlNode xmlNode)
33983411
{
33993412
EI.X = Convert.ToInt32(boundsXY[0]);
34003413
EI.Y = Convert.ToInt32(boundsXY[1]);
3414+
EI.Width = Math.Max(0, Convert.ToInt32(boundsXY[2]) - EI.X);
3415+
EI.Height = Math.Max(0, Convert.ToInt32(boundsXY[3]) - EI.Y);
34013416
}
34023417
}
34033418

0 commit comments

Comments
 (0)