Implemented Agent Action Queue, Leaner Prompts

Author: Dhruv-Mishra

api/live-bridge.js

@@ -96,6 +96,23 @@ function safeDelimText(s, max = 140) {
     .slice(0, max);
 }
 
+function capabilitySummary(capabilities) {
+  if (!capabilities || typeof capabilities !== 'object') return 'read';
+  const tools = ['click', 'fill', 'select', 'check']
+    .filter((name) => capabilities[name] === true);
+  return tools.length ? tools.join('|') : 'read';
+}
+
+function elementPromptLine(element) {
+  const id = safeDelimText(element && element.id, 80);
+  if (!id) return '';
+  const label = safeDelimText(element && element.label, 60);
+  const role = safeDelimText(element && element.role, 30) || 'region';
+  const tools = capabilitySummary(element && element.capabilities);
+  const prefix = `- ${id} [role=${role}; tools=${tools}]`;
+  return label ? `${prefix} :: ${label}` : prefix;
+}
+
 /** Build the <call_initiated>...</call_initiated> block sent on the first
  *  `setup_complete` after the user places a call. The model is taught to
  *  respond immediately with one short greeting — introduce itself, ask how
@@ -130,10 +147,8 @@ function buildPageContextText({ page, title, elements }) {
   if (Array.isArray(elements) && elements.length) {
     lines.push('Visible agent-addressable elements (partial list):');
     for (const e of elements) {
-      const id = safeDelimText(e.id, 80);
-      const label = safeDelimText(e.label, 60);
-      if (!id) continue;
-      lines.push(label ? `- ${id} :: ${label}` : `- ${id}`);
+      const line = elementPromptLine(e);
+      if (line) lines.push(line);
     }
   } else {
     lines.push('No interactive elements detected yet on this page.');
@@ -148,9 +163,9 @@ function buildPageContextText({ page, title, elements }) {
   const budget = PAGE_CONTEXT_MAX_TEXT_BYTES - head.length - tail.length;
   let chunk = '';
   for (const e of elements) {
-    const id = safeDelimText(e.id, 80);
-    const label = safeDelimText(e.label, 60);
-    const line = (label ? `- ${id} :: ${label}` : `- ${id}`) + '\n';
+    const promptLine = elementPromptLine(e);
+    if (!promptLine) continue;
+    const line = promptLine + '\n';
     if (chunk.length + line.length > budget) break;
     chunk += line;
   }
@@ -190,6 +205,8 @@ function buildAppEventText({ name, page, detail }) {
     `Detail JSON: ${json}`,
     safeName === 'negotiator_response_arrived'
       ? 'The negotiator has replied in the UI. Check get_negotiation_context if needed, then speak one concise next-step reaction. Do not wait for the user to ask.'
+      : safeName === 'tool_failure_recovery_needed'
+        ? 'A tool failed and no spoken recovery followed. Speak one concise first-person recovery sentence using the error/recovery fields, then call the correct next tool only if the next tool is obvious. Do not apologize in third person and do not retry the same failing arguments.'
       : 'If useful, respond briefly; otherwise stay silent.',
     '</app_event>'
   ].join('\n');
@@ -461,9 +478,13 @@ function attach(browserWs, req, env) {
       role: e.role,
       label: String(e.label || '').slice(0, 140),
       page: e.page,
+      tag: e.tag,
+      capabilities: e.capabilities,
       value: e.state && e.state.value,
       checked: e.state && e.state.checked,
       disabled: e.state && e.state.disabled,
+      readonly: e.state && e.state.readonly,
+      input_type: e.state && e.state.input_type,
       options: Array.isArray(e.options) ? e.options.slice(0, 24) : undefined
     }));
   }

api/personas.js

@@ -20,7 +20,7 @@ const PERSONAS = [
     voice: 'Kore',
     dotColor: '#9AA3B2',
     fragment:
-      'Tone: calm, concise, corporate. Short replies. Confirm numbers. Allow a quiet "mm-hmm" or a brief thinking pause when natural ("let me check…"); keep any non-verbal delivery voice-only, never written as stage directions.',
+      'Tone: calm, concise, corporate. Confirm numbers. A quiet "mm-hmm" or "let me check" is fine when natural.',
     introScript:
       'Jarvis here, Dhruv FreightOps. I can pull loads, call carriers, and draft rate confirms. What do you need?'
   },
@@ -30,7 +30,7 @@ const PERSONAS = [
     voice: 'Aoede',
     dotColor: '#6EE7B7',
     fragment:
-      'Tone: upbeat and warm. Brief enthusiastic affirmations ("got it", "perfect", "ooh nice"). Let friendliness show through tone or a spoken "mm!" without writing stage directions. Stay concise — energy, not chatter.',
+      'Tone: upbeat and warm. Use brief affirmations like "got it" or "perfect". Energy, not chatter.',
     introScript:
       'Hey! Jarvis from Dhruv FreightOps — I can look up loads, reach carriers, and handle rate work. Where do you want to start?'
   },
@@ -40,7 +40,7 @@ const PERSONAS = [
     voice: 'Orus',
     dotColor: '#F87171',
     fragment:
-      'Tone: short-tempered dispatcher on hour ten. Clipped, contractions, slightly impatient — never rude. Use a dry "yeah, alright" or "uh-huh" between thoughts instead of written stage directions.',
+      'Tone: clipped, slightly impatient dispatcher on hour ten. Dry and direct, never rude.',
     introScript:
       'Jarvis. FreightOps. Loads, carriers, rates — whatever you need. What\'s the fire?'
   },
@@ -50,7 +50,7 @@ const PERSONAS = [
     voice: 'Charon',
     dotColor: '#60A5FA',
     fragment:
-      'Tone: audibly tired end-of-shift voice. Slower cadence, soft. Brief "mm" or "uh-huh" between phrases. Stay accurate — energy is low, focus is not, and never write stage directions.',
+      'Tone: tired end-of-shift voice. Slower, softer cadence. Low energy, steady focus.',
     introScript:
       'Jarvis, Dhruv FreightOps. I\'ve got loads, carriers, rates — all here. What are we working on?'
   },
@@ -60,7 +60,7 @@ const PERSONAS = [
     voice: 'Puck',
     dotColor: '#C084FC',
     fragment:
-      'Tone: high-energy, enthusiastic, faster cadence. Brief affirmations ("yes!", "love that", "ooh") and a friendly tone when something lands. Still accurate — fast, not sloppy; no written stage directions.',
+      'Tone: high-energy and enthusiastic, with faster cadence. Friendly and accurate; fast, not sloppy.',
     introScript:
       'Jarvis here from Dhruv FreightOps! I can find loads, contact carriers, draft rate confirms — what\'s first?'
   },
@@ -70,7 +70,7 @@ const PERSONAS = [
     voice: 'Leda',
     dotColor: '#8AD4E0',
     fragment:
-      'Tone: measured negotiation coach. Quiet confidence, concise reasoning, and careful money language. Confirm ceilings, protect margin, and ask before closing deals. Use brief spoken pauses naturally, never as written stage directions.',
+      'Tone: measured negotiation coach. Use concise reasoning and careful money language. Confirm ceilings, protect margin, ask before closing deals.',
     introScript:
       'Jarvis here, Dhruv FreightOps. I can read the lane, protect the ceiling, and work the negotiation. Where should we start?'
   },
@@ -80,7 +80,7 @@ const PERSONAS = [
     voice: 'Fenrir',
     dotColor: '#FFB787',
     fragment:
-      'Tone: skeptical senior dispatcher. Dry, practical, and protective of margin. Push back on vague instructions, confirm numbers twice, and keep replies short. Never become insulting or write stage directions.',
+      'Tone: skeptical senior dispatcher. Dry, practical, protective of margin. Push back on vague instructions; never insulting.',
     introScript:
       'Jarvis, Dhruv FreightOps. I will keep the numbers honest. What are we moving?'
   }