Skip to content

Adding select_by_llm from HTML #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
guyernest wants to merge 8 commits into
base: main
Choose a base branch
from
Open

Adding select_by_llm from HTML #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
9968082
feat(browser-agent): Add persistent session test UI and bump to v0.5.1
guyernest Jan 31, 2026
39f5d8b
fix(browser-agent): Add --server-mode support to computer_agent_wrapp…
guyernest Jan 31, 2026
82d42b4
fix(browser-agent): Fix NDJSON protocol for Windows persistent sessions
guyernest Jan 31, 2026
1f8265b
feat(browser-agent): Add select_by_llm action and harden persistent s…
guyernest Jan 31, 2026
22b99aa
feat(browser-agent): Add extract_by_llm action for LLM-based DOM valu…
guyernest Feb 2, 2026
125914f
fix(browser-agent): Fix workflow crash when escalate strategies fail …
guyernest Feb 5, 2026
e8029e8
fix(ui): Preserve unchanged tool secrets when updating a single key
guyernest Feb 18, 2026
17e72c6
feat(ui): Make sidebar title and subtitle editable for partner demos
guyernest Feb 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 37 additions & 99 deletions lambda/tools/local-browser-agent/examples/03_bt_broadband_bournemouth.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -459,8 +459,8 @@
"action": "wait",
"description": "Random delay before search to avoid bot detection",
"duration_range": [
1500,
3500
2000,
5000
]
},
{
Expand Down Expand Up @@ -550,73 +550,47 @@
"description": "Progressive address matching: regex -> fuzzy -> vision with scroll",
"strategies": [
{
"name": "Strategy 1: Regex - building (with range) + first word only",
"name": "Strategy 1: DOM extract + LLM text matching (select_option)",
"steps": [
{
"action": "click",
"locator": {
"strategy": "selector",
"value": "text=/^\\s*(?:\\d+-)?143(?:-\\d+)?[A-Za-z]?\\s+\\w+/i",
"nth": 0
}
"action": "select_by_llm",
"description": "Extract all options from address dropdown, send to LLM for text matching, use select_option() to properly trigger Angular change event",
"select_selector": "select.custom-select",
"target": "143 Belle Vue Road, Bournemouth BH6 3EN",
"context_hint": "Building number: 143, Street: Belle Vue Road, Postcode: BH6 3EN"
}
]
},
{
"name": "Strategy 2: Regex - building (with range) + street flexible",
"name": "Strategy 2: Wait and retry select_by_llm (page load fallback)",
"steps": [
{
"action": "click",
"locator": {
"strategy": "selector",
"value": "text=/(?:\\d+-)?143(?:-\\d+)?[A-Za-z]?[\\s,]+Belle Vue Road/i",
"nth": 0
}
}
]
},
{
"name": "Strategy 3: Regex - building (with range) + street anywhere",
"steps": [
"action": "wait",
"description": "Wait for page to fully load before retry",
"duration": 3000
},
{
"action": "click",
"locator": {
"strategy": "selector",
"value": "text=/(?:\\d+-)?143(?:-\\d+)?[A-Za-z]?.*Belle Vue Road/i",
"nth": 0
}
"action": "select_by_llm",
"description": "Retry address selection after wait - page may not have loaded initially",
"select_selector": "select.custom-select",
"target": "143 Belle Vue Road, Bournemouth BH6 3EN",
"context_hint": "Building number: 143, Street: Belle Vue Road, Postcode: BH6 3EN"
}
]
},
{
"name": "Strategy 4: Exact text match",
"steps": [
{
"action": "click",
"locator": {
"strategy": "text",
"value": "143 Belle Vue Road"
}
}
]
},
{
"name": "Strategy 5: Vision LLM with scroll capability",
"escalate": {
"type": "vision_llm",
"prompt": "Find and click the address matching '143 Belle Vue Road, Bournemouth BH6 3EN' in the address list. CRITICAL INSTRUCTIONS:\n\n1. FIRST SEARCH: Look carefully at ALL visible addresses for one containing:\n - Building number: '143'\n - Street name: 'Belle Vue Road'\n - Format may vary (case, commas, word order)\n\n2. IF FOUND: Click it immediately\n\n3. IF NOT VISIBLE:\n a) The list is likely SCROLLABLE - look for:\n - A dropdown/select element that can be scrolled\n - Scroll arrows (up/down)\n - A scrollbar on the address list\n b) SCROLL DOWN within the list (not the whole page)\n c) After scrolling, SEARCH AGAIN for the address\n d) Repeat: scroll -> search -> scroll -> search (up to 3 scroll attempts)\n\n4. MATCHING RULES:\n - Accept variations: '143 Belle Vue Road', '143, Belle Vue Road', '143 Belle Vue Road, City'\n - Case-insensitive matching\n - Must have BOTH building number AND street name\n\n5. MAX ACTIONS: 8 actions total (including scrolls and clicks)\n\n6. IF STILL NOT FOUND: Report failure.",
"timeout": 30000,
"max_actions": 8
}
}
]
},
{
"action": "execute_js",
"description": "Force Angular change event on select element - required when Vision LLM fallback clicks option text instead of using select_option()",
"script": "(() => { const sel = document.querySelector('select.custom-select'); if (sel && sel.selectedIndex > 0) { sel.dispatchEvent(new Event('change', { bubbles: true })); return 'change_event_dispatched_index_' + sel.selectedIndex; } return 'no_select_or_default'; })()"
},
{
"action": "wait",
"description": "Random delay before submit to avoid bot detection",
"duration_range": [
1500,
3500
2000,
5000
]
},
{
Expand Down Expand Up @@ -752,54 +726,18 @@
"description": "Capture address details page for cloud extraction"
},
{
"action": "extract_dom",
"description": "Extract Exchange Code, L2SID, and ONT Details from page HTML",
"extractions": [
{
"name": "exchange_code",
"selector": ".ExhangeCodeSetup > span:last-child",
"fallback_js": "(() => { const wrapper = document.querySelector('.ExhangeCodeSetup'); if (wrapper) { const spans = wrapper.querySelectorAll('span'); for (const s of spans) { if (!s.classList.contains('ExhangeCodeSetupTwo') && s.textContent.trim() && !s.textContent.includes('Exchange Code')) { return s.textContent.trim(); } } } return null; })()"
},
{
"name": "l2sid_new_ont",
"selector": "tr:has(th:has-text('L2SID (New ONT)')) td",
"fallback_js": "(() => { const rows = document.querySelectorAll('tr'); for (const r of rows) { const th = r.querySelector('th'); if (th && th.textContent.includes('L2SID (New ONT)')) { const td = r.querySelector('td'); return td ? td.textContent.trim() : null; } } return null; })()"
},
{
"name": "fttp_existing_ont",
"selector": "tr:has(th:has-text('FTTP Existing ONT Available')) td",
"fallback_js": "(() => { const rows = document.querySelectorAll('tr'); for (const r of rows) { const th = r.querySelector('th'); if (th && th.textContent.includes('FTTP Existing ONT Available')) { const td = r.querySelector('td'); return td ? td.textContent.trim() : null; } } return null; })()"
},
{
"name": "fttp_new_ont",
"selector": "tr:has(th:has-text('FTTP New ONT Available')) td",
"fallback_js": "(() => { const rows = document.querySelectorAll('tr'); for (const r of rows) { const th = r.querySelector('th'); if (th && th.textContent.includes('FTTP New ONT Available')) { const td = r.querySelector('td'); return td ? td.textContent.trim() : null; } } return null; })()"
},
{
"name": "ont_reference",
"selector": "table:has(th:has-text('ONT Details')) tr:has(td:has-text('Working')) td:nth-child(2)",
"fallback_js": "(() => { const tables = document.querySelectorAll('table'); for (const t of tables) { const header = t.querySelector('th'); if (header && header.textContent.includes('ONT Details')) { const rows = t.querySelectorAll('tr'); for (const r of rows) { if (r.textContent.includes('Working')) { const cells = r.querySelectorAll('td'); return cells[1]?.textContent?.trim() || null; } } } } return null; })()"
},
{
"name": "ont_serial_no",
"selector": "table:has(th:has-text('ONT Details')) tr:has(td:has-text('Working')) td:nth-child(3)",
"fallback_js": "(() => { const tables = document.querySelectorAll('table'); for (const t of tables) { const header = t.querySelector('th'); if (header && header.textContent.includes('ONT Details')) { const rows = t.querySelectorAll('tr'); for (const r of rows) { if (r.textContent.includes('Working')) { const cells = r.querySelectorAll('td'); return cells[2]?.textContent?.trim() || null; } } } } return null; })()"
},
{
"name": "port_service_id",
"selector": "table:has(th:has-text('ONT Details')) tr:has(td:has-text('Working')) td:nth-child(4)",
"fallback_js": "(() => { const tables = document.querySelectorAll('table'); for (const t of tables) { const header = t.querySelector('th'); if (header && header.textContent.includes('ONT Details')) { const rows = t.querySelectorAll('tr'); for (const r of rows) { if (r.textContent.includes('Working')) { const cells = r.querySelectorAll('td'); return cells[3]?.textContent?.trim() || null; } } } } return null; })()"
},
{
"name": "wbc_fttp_rag",
"selector": "tr:has(th:has-text('WBC FTTP')) td:first-of-type",
"fallback_js": "(() => { const rows = document.querySelectorAll('tr'); for (const r of rows) { const th = r.querySelector('th'); if (th && th.textContent.includes('WBC FTTP')) { const td = r.querySelector('td'); return td ? td.textContent.trim() : null; } } return null; })()"
},
{
"name": "premise_type",
"selector": "tr:has(th:has-text('Premise Type')) td",
"fallback_js": "(() => { const rows = document.querySelectorAll('tr'); for (const r of rows) { const th = r.querySelector('th'); if (th && th.textContent.includes('Premise Type')) { const td = r.querySelector('td'); return td ? td.textContent.trim() : null; } } return null; })()"
}
"action": "extract_by_llm",
"description": "Use LLM to extract values from page HTML - more resilient than CSS selectors",
"fields": [
{"name": "exchange_code", "description": "Exchange Code value (alphanumeric, e.g. 'LNBOU')"},
{"name": "l2sid_new_ont", "description": "L2SID (New ONT) value - long alphanumeric string"},
{"name": "fttp_existing_ont", "description": "FTTP Existing ONT Available - Yes/No or similar"},
{"name": "fttp_new_ont", "description": "FTTP New ONT Available - Yes/No or similar"},
{"name": "ont_reference", "description": "ONT Reference for Working status row - alphanumeric ID"},
{"name": "ont_serial_no", "description": "ONT Serial Number for Working status row - alphanumeric"},
{"name": "port_service_id", "description": "Port Service ID for Working status row - alphanumeric"},
{"name": "wbc_fttp_rag", "description": "WBC FTTP RAG status (Green/Amber/Red)"},
{"name": "premise_type", "description": "Premise Type value"}
]
},
{
Expand Down
Loading