Skip to content

Commit 22b99aa

Browse files
guyernestclaude
guyernest
and
claude
committed
feat(browser-agent): Add extract_by_llm action for LLM-based DOM value extraction (v0.5.4)
Replace CSS-selector-based extract_dom (72% failure rate) with extract_by_llm that sends cleaned page HTML to the LLM for field extraction. Strips non-content elements, truncates to 50K chars, and uses JSON mode for structured output. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1 parent 1f8265b commit 22b99aa

7 files changed

Lines changed: 291 additions & 335 deletions

File tree

‎lambda/tools/local-browser-agent/examples/03_bt_broadband_bournemouth.json‎

Lines changed: 23 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -459,8 +459,8 @@
459459
"action": "wait",
460460
"description": "Random delay before search to avoid bot detection",
461461
"duration_range": [
462-
1500,
463-
3500
462+
2000,
463+
5000
464464
]
465465
},
466466
{
@@ -550,20 +550,7 @@
550550
"description": "Progressive address matching: regex -> fuzzy -> vision with scroll",
551551
"strategies": [
552552
{
553-
"name": "Strategy 1: Regex - building number + first word",
554-
"steps": [
555-
{
556-
"action": "click",
557-
"locator": {
558-
"strategy": "selector",
559-
"value": "text=/^\\s*(?:\\d+-)?143(?:-\\d+)?[A-Za-z]?\\s+\\w+/i",
560-
"nth": 0
561-
}
562-
}
563-
]
564-
},
565-
{
566-
"name": "Strategy 2: DOM extract + LLM text matching",
553+
"name": "Strategy 1: DOM extract + LLM text matching (select_option)",
567554
"steps": [
568555
{
569556
"action": "select_by_llm",
@@ -575,7 +562,7 @@
575562
]
576563
},
577564
{
578-
"name": "Strategy 3: Vision LLM with scroll capability (fallback)",
565+
"name": "Strategy 2: Vision LLM with scroll capability (fallback)",
579566
"escalate": {
580567
"type": "vision_llm",
581568
"prompt": "Find and click the address matching '143 Belle Vue Road, Bournemouth BH6 3EN' in the address list. CRITICAL INSTRUCTIONS:\n\n1. FIRST SEARCH: Look carefully at ALL visible addresses for one containing:\n - Building number: '143'\n - Street name: 'Belle Vue Road'\n - Format may vary (case, commas, word order)\n\n2. IF FOUND: Click it immediately\n\n3. IF NOT VISIBLE:\n a) The list is likely SCROLLABLE - look for:\n - A dropdown/select element that can be scrolled\n - Scroll arrows (up/down)\n - A scrollbar on the address list\n b) SCROLL DOWN within the list (not the whole page)\n c) After scrolling, SEARCH AGAIN for the address\n d) Repeat: scroll -> search -> scroll -> search (up to 3 scroll attempts)\n\n4. MATCHING RULES:\n - Accept variations: '143 Belle Vue Road', '143, Belle Vue Road', '143 Belle Vue Road, City'\n - Case-insensitive matching\n - Must have BOTH building number AND street name\n\n5. MAX ACTIONS: 8 actions total (including scrolls and clicks)\n\n6. IF STILL NOT FOUND: Report failure.",
@@ -585,12 +572,17 @@
585572
}
586573
]
587574
},
575+
{
576+
"action": "execute_js",
577+
"description": "Force Angular change event on select element - required when Vision LLM fallback clicks option text instead of using select_option()",
578+
"script": "(() => { const sel = document.querySelector('select.custom-select'); if (sel && sel.selectedIndex > 0) { sel.dispatchEvent(new Event('change', { bubbles: true })); return 'change_event_dispatched_index_' + sel.selectedIndex; } return 'no_select_or_default'; })()"
579+
},
588580
{
589581
"action": "wait",
590582
"description": "Random delay before submit to avoid bot detection",
591583
"duration_range": [
592-
1500,
593-
3500
584+
2000,
585+
5000
594586
]
595587
},
596588
{
@@ -726,54 +718,18 @@
726718
"description": "Capture address details page for cloud extraction"
727719
},
728720
{
729-
"action": "extract_dom",
730-
"description": "Extract Exchange Code, L2SID, and ONT Details from page HTML",
731-
"extractions": [
732-
{
733-
"name": "exchange_code",
734-
"selector": ".ExhangeCodeSetup > span:last-child",
735-
"fallback_js": "(() => { const wrapper = document.querySelector('.ExhangeCodeSetup'); if (wrapper) { const spans = wrapper.querySelectorAll('span'); for (const s of spans) { if (!s.classList.contains('ExhangeCodeSetupTwo') && s.textContent.trim() && !s.textContent.includes('Exchange Code')) { return s.textContent.trim(); } } } return null; })()"
736-
},
737-
{
738-
"name": "l2sid_new_ont",
739-
"selector": "tr:has(th:has-text('L2SID (New ONT)')) td",
740-
"fallback_js": "(() => { const rows = document.querySelectorAll('tr'); for (const r of rows) { const th = r.querySelector('th'); if (th && th.textContent.includes('L2SID (New ONT)')) { const td = r.querySelector('td'); return td ? td.textContent.trim() : null; } } return null; })()"
741-
},
742-
{
743-
"name": "fttp_existing_ont",
744-
"selector": "tr:has(th:has-text('FTTP Existing ONT Available')) td",
745-
"fallback_js": "(() => { const rows = document.querySelectorAll('tr'); for (const r of rows) { const th = r.querySelector('th'); if (th && th.textContent.includes('FTTP Existing ONT Available')) { const td = r.querySelector('td'); return td ? td.textContent.trim() : null; } } return null; })()"
746-
},
747-
{
748-
"name": "fttp_new_ont",
749-
"selector": "tr:has(th:has-text('FTTP New ONT Available')) td",
750-
"fallback_js": "(() => { const rows = document.querySelectorAll('tr'); for (const r of rows) { const th = r.querySelector('th'); if (th && th.textContent.includes('FTTP New ONT Available')) { const td = r.querySelector('td'); return td ? td.textContent.trim() : null; } } return null; })()"
751-
},
752-
{
753-
"name": "ont_reference",
754-
"selector": "table:has(th:has-text('ONT Details')) tr:has(td:has-text('Working')) td:nth-child(2)",
755-
"fallback_js": "(() => { const tables = document.querySelectorAll('table'); for (const t of tables) { const header = t.querySelector('th'); if (header && header.textContent.includes('ONT Details')) { const rows = t.querySelectorAll('tr'); for (const r of rows) { if (r.textContent.includes('Working')) { const cells = r.querySelectorAll('td'); return cells[1]?.textContent?.trim() || null; } } } } return null; })()"
756-
},
757-
{
758-
"name": "ont_serial_no",
759-
"selector": "table:has(th:has-text('ONT Details')) tr:has(td:has-text('Working')) td:nth-child(3)",
760-
"fallback_js": "(() => { const tables = document.querySelectorAll('table'); for (const t of tables) { const header = t.querySelector('th'); if (header && header.textContent.includes('ONT Details')) { const rows = t.querySelectorAll('tr'); for (const r of rows) { if (r.textContent.includes('Working')) { const cells = r.querySelectorAll('td'); return cells[2]?.textContent?.trim() || null; } } } } return null; })()"
761-
},
762-
{
763-
"name": "port_service_id",
764-
"selector": "table:has(th:has-text('ONT Details')) tr:has(td:has-text('Working')) td:nth-child(4)",
765-
"fallback_js": "(() => { const tables = document.querySelectorAll('table'); for (const t of tables) { const header = t.querySelector('th'); if (header && header.textContent.includes('ONT Details')) { const rows = t.querySelectorAll('tr'); for (const r of rows) { if (r.textContent.includes('Working')) { const cells = r.querySelectorAll('td'); return cells[3]?.textContent?.trim() || null; } } } } return null; })()"
766-
},
767-
{
768-
"name": "wbc_fttp_rag",
769-
"selector": "tr:has(th:has-text('WBC FTTP')) td:first-of-type",
770-
"fallback_js": "(() => { const rows = document.querySelectorAll('tr'); for (const r of rows) { const th = r.querySelector('th'); if (th && th.textContent.includes('WBC FTTP')) { const td = r.querySelector('td'); return td ? td.textContent.trim() : null; } } return null; })()"
771-
},
772-
{
773-
"name": "premise_type",
774-
"selector": "tr:has(th:has-text('Premise Type')) td",
775-
"fallback_js": "(() => { const rows = document.querySelectorAll('tr'); for (const r of rows) { const th = r.querySelector('th'); if (th && th.textContent.includes('Premise Type')) { const td = r.querySelector('td'); return td ? td.textContent.trim() : null; } } return null; })()"
776-
}
721+
"action": "extract_by_llm",
722+
"description": "Use LLM to extract values from page HTML - more resilient than CSS selectors",
723+
"fields": [
724+
{"name": "exchange_code", "description": "Exchange Code value (alphanumeric, e.g. 'LNBOU')"},
725+
{"name": "l2sid_new_ont", "description": "L2SID (New ONT) value - long alphanumeric string"},
726+
{"name": "fttp_existing_ont", "description": "FTTP Existing ONT Available - Yes/No or similar"},
727+
{"name": "fttp_new_ont", "description": "FTTP New ONT Available - Yes/No or similar"},
728+
{"name": "ont_reference", "description": "ONT Reference for Working status row - alphanumeric ID"},
729+
{"name": "ont_serial_no", "description": "ONT Serial Number for Working status row - alphanumeric"},
730+
{"name": "port_service_id", "description": "Port Service ID for Working status row - alphanumeric"},
731+
{"name": "wbc_fttp_rag", "description": "WBC FTTP RAG status (Green/Amber/Red)"},
732+
{"name": "premise_type", "description": "Premise Type value"}
777733
]
778734
},
779735
{

‎lambda/tools/local-browser-agent/examples/04_bt_broadband_bolton.json‎

Lines changed: 23 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -459,8 +459,8 @@
459459
"action": "wait",
460460
"description": "Random delay before search to avoid bot detection",
461461
"duration_range": [
462-
1500,
463-
3500
462+
2000,
463+
5000
464464
]
465465
},
466466
{
@@ -550,20 +550,7 @@
550550
"description": "Progressive address matching: regex -> fuzzy -> vision with scroll",
551551
"strategies": [
552552
{
553-
"name": "Strategy 1: Regex - building number + first word",
554-
"steps": [
555-
{
556-
"action": "click",
557-
"locator": {
558-
"strategy": "selector",
559-
"value": "text=/^\\s*(?:\\d+-)?34(?:-\\d+)?[A-Za-z]?\\s+\\w+/i",
560-
"nth": 0
561-
}
562-
}
563-
]
564-
},
565-
{
566-
"name": "Strategy 2: DOM extract + LLM text matching",
553+
"name": "Strategy 1: DOM extract + LLM text matching (select_option)",
567554
"steps": [
568555
{
569556
"action": "select_by_llm",
@@ -575,7 +562,7 @@
575562
]
576563
},
577564
{
578-
"name": "Strategy 3: Vision LLM with scroll capability (fallback)",
565+
"name": "Strategy 2: Vision LLM with scroll capability (fallback)",
579566
"escalate": {
580567
"type": "vision_llm",
581568
"prompt": "Find and click the address matching '34-40 Market Street, Westhoughton, Bolton BL5 3AN' in the address list. CRITICAL INSTRUCTIONS:\n\n1. FIRST SEARCH: Look carefully at ALL visible addresses for one containing:\n - Building number: '34'\n - Street name: 'Market Street'\n - Format may vary (case, commas, word order)\n\n2. IF FOUND: Click it immediately\n\n3. IF NOT VISIBLE:\n a) The list is likely SCROLLABLE - look for:\n - A dropdown/select element that can be scrolled\n - Scroll arrows (up/down)\n - A scrollbar on the address list\n b) SCROLL DOWN within the list (not the whole page)\n c) After scrolling, SEARCH AGAIN for the address\n d) Repeat: scroll -> search -> scroll -> search (up to 3 scroll attempts)\n\n4. MATCHING RULES:\n - Accept variations: '34 Market Street', '34, Market Street', '34 Market Street, City'\n - Case-insensitive matching\n - Must have BOTH building number AND street name\n\n5. MAX ACTIONS: 8 actions total (including scrolls and clicks)\n\n6. IF STILL NOT FOUND: Report failure.",
@@ -585,12 +572,17 @@
585572
}
586573
]
587574
},
575+
{
576+
"action": "execute_js",
577+
"description": "Force Angular change event on select element - required when Vision LLM fallback clicks option text instead of using select_option()",
578+
"script": "(() => { const sel = document.querySelector('select.custom-select'); if (sel && sel.selectedIndex > 0) { sel.dispatchEvent(new Event('change', { bubbles: true })); return 'change_event_dispatched_index_' + sel.selectedIndex; } return 'no_select_or_default'; })()"
579+
},
588580
{
589581
"action": "wait",
590582
"description": "Random delay before submit to avoid bot detection",
591583
"duration_range": [
592-
1500,
593-
3500
584+
2000,
585+
5000
594586
]
595587
},
596588
{
@@ -726,54 +718,18 @@
726718
"description": "Capture address details page for cloud extraction"
727719
},
728720
{
729-
"action": "extract_dom",
730-
"description": "Extract Exchange Code, L2SID, and ONT Details from page HTML",
731-
"extractions": [
732-
{
733-
"name": "exchange_code",
734-
"selector": ".ExhangeCodeSetup > span:last-child",
735-
"fallback_js": "(() => { const wrapper = document.querySelector('.ExhangeCodeSetup'); if (wrapper) { const spans = wrapper.querySelectorAll('span'); for (const s of spans) { if (!s.classList.contains('ExhangeCodeSetupTwo') && s.textContent.trim() && !s.textContent.includes('Exchange Code')) { return s.textContent.trim(); } } } return null; })()"
736-
},
737-
{
738-
"name": "l2sid_new_ont",
739-
"selector": "tr:has(th:has-text('L2SID (New ONT)')) td",
740-
"fallback_js": "(() => { const rows = document.querySelectorAll('tr'); for (const r of rows) { const th = r.querySelector('th'); if (th && th.textContent.includes('L2SID (New ONT)')) { const td = r.querySelector('td'); return td ? td.textContent.trim() : null; } } return null; })()"
741-
},
742-
{
743-
"name": "fttp_existing_ont",
744-
"selector": "tr:has(th:has-text('FTTP Existing ONT Available')) td",
745-
"fallback_js": "(() => { const rows = document.querySelectorAll('tr'); for (const r of rows) { const th = r.querySelector('th'); if (th && th.textContent.includes('FTTP Existing ONT Available')) { const td = r.querySelector('td'); return td ? td.textContent.trim() : null; } } return null; })()"
746-
},
747-
{
748-
"name": "fttp_new_ont",
749-
"selector": "tr:has(th:has-text('FTTP New ONT Available')) td",
750-
"fallback_js": "(() => { const rows = document.querySelectorAll('tr'); for (const r of rows) { const th = r.querySelector('th'); if (th && th.textContent.includes('FTTP New ONT Available')) { const td = r.querySelector('td'); return td ? td.textContent.trim() : null; } } return null; })()"
751-
},
752-
{
753-
"name": "ont_reference",
754-
"selector": "table:has(th:has-text('ONT Details')) tr:has(td:has-text('Working')) td:nth-child(2)",
755-
"fallback_js": "(() => { const tables = document.querySelectorAll('table'); for (const t of tables) { const header = t.querySelector('th'); if (header && header.textContent.includes('ONT Details')) { const rows = t.querySelectorAll('tr'); for (const r of rows) { if (r.textContent.includes('Working')) { const cells = r.querySelectorAll('td'); return cells[1]?.textContent?.trim() || null; } } } } return null; })()"
756-
},
757-
{
758-
"name": "ont_serial_no",
759-
"selector": "table:has(th:has-text('ONT Details')) tr:has(td:has-text('Working')) td:nth-child(3)",
760-
"fallback_js": "(() => { const tables = document.querySelectorAll('table'); for (const t of tables) { const header = t.querySelector('th'); if (header && header.textContent.includes('ONT Details')) { const rows = t.querySelectorAll('tr'); for (const r of rows) { if (r.textContent.includes('Working')) { const cells = r.querySelectorAll('td'); return cells[2]?.textContent?.trim() || null; } } } } return null; })()"
761-
},
762-
{
763-
"name": "port_service_id",
764-
"selector": "table:has(th:has-text('ONT Details')) tr:has(td:has-text('Working')) td:nth-child(4)",
765-
"fallback_js": "(() => { const tables = document.querySelectorAll('table'); for (const t of tables) { const header = t.querySelector('th'); if (header && header.textContent.includes('ONT Details')) { const rows = t.querySelectorAll('tr'); for (const r of rows) { if (r.textContent.includes('Working')) { const cells = r.querySelectorAll('td'); return cells[3]?.textContent?.trim() || null; } } } } return null; })()"
766-
},
767-
{
768-
"name": "wbc_fttp_rag",
769-
"selector": "tr:has(th:has-text('WBC FTTP')) td:first-of-type",
770-
"fallback_js": "(() => { const rows = document.querySelectorAll('tr'); for (const r of rows) { const th = r.querySelector('th'); if (th && th.textContent.includes('WBC FTTP')) { const td = r.querySelector('td'); return td ? td.textContent.trim() : null; } } return null; })()"
771-
},
772-
{
773-
"name": "premise_type",
774-
"selector": "tr:has(th:has-text('Premise Type')) td",
775-
"fallback_js": "(() => { const rows = document.querySelectorAll('tr'); for (const r of rows) { const th = r.querySelector('th'); if (th && th.textContent.includes('Premise Type')) { const td = r.querySelector('td'); return td ? td.textContent.trim() : null; } } return null; })()"
776-
}
721+
"action": "extract_by_llm",
722+
"description": "Use LLM to extract values from page HTML - more resilient than CSS selectors",
723+
"fields": [
724+
{"name": "exchange_code", "description": "Exchange Code value (alphanumeric, e.g. 'LNBOU')"},
725+
{"name": "l2sid_new_ont", "description": "L2SID (New ONT) value - long alphanumeric string"},
726+
{"name": "fttp_existing_ont", "description": "FTTP Existing ONT Available - Yes/No or similar"},
727+
{"name": "fttp_new_ont", "description": "FTTP New ONT Available - Yes/No or similar"},
728+
{"name": "ont_reference", "description": "ONT Reference for Working status row - alphanumeric ID"},
729+
{"name": "ont_serial_no", "description": "ONT Serial Number for Working status row - alphanumeric"},
730+
{"name": "port_service_id", "description": "Port Service ID for Working status row - alphanumeric"},
731+
{"name": "wbc_fttp_rag", "description": "WBC FTTP RAG status (Green/Amber/Red)"},
732+
{"name": "premise_type", "description": "Premise Type value"}
777733
]
778734
},
779735
{

0 commit comments

Comments
 (0)