feat(companion): report first visible subtitle

catalog/catalog.json

@@ -145,6 +145,23 @@
         "voice"
       ],
       "versions": [
+        {
+          "version": "0.1.0-alpha.23",
+          "contractVersion": "0.1.0",
+          "manifestPath": "plugin.yaml",
+          "artifact": {
+            "kind": "npm",
+            "packageName": "@clowder-ai/companion",
+            "version": "0.1.0-alpha.23",
+            "tarballUrl": "https://registry.npmjs.org/@clowder-ai/companion/-/companion-0.1.0-alpha.23.tgz",
+            "integrity": "sha512-uJimgsfGi9p6DROxQYFlB3CYj3mwkmTqHJzaR/NkGslLqE5LgXjJ6FLkORr0DEPnJjNoALyk2DDkhQ+Cg0XkTw==",
+            "shasum": "5f5aaa9e51ef864db131b7af96450e4848c6652c",
+            "provenance": {
+              "repository": "https://github.com/zts212653/clowder-ai-plugins",
+              "sourceDirectory": "packages/companion"
+            }
+          }
+        },
         {
           "version": "0.1.0-alpha.22",
           "contractVersion": "0.1.0",

packages/companion/README.md

@@ -26,12 +26,21 @@ borrowing today's selected companion.
 
 ## Host integration
 
-This surface requires the additive desktop bridge **1.3.0**. The Host computes
+This surface requires the additive desktop bridge **1.4.0**. The Host computes
 placement within the display work area, keeps the cat as its stable anchor, and
-owns bounded drag gestures and transparent-area hit testing. Older 1.0-only
-Hosts reject this manifest before installation. Media admission and capture
+owns bounded drag gestures and transparent-area hit testing. Hosts below 1.4
+reject this manifest before installation. Media admission and capture
 remain in the trusted Host; this package receives neither streams nor provider
 SDP. Hiding releases media; the existing Clowder **聊聊** entry restores the cat.
+Bridge 1.4 adds one untrusted first-visible-subtitle observation containing only
+an epoch-millisecond time, the current call fence and a Host-correlatable source
+reference. It does not claim compositor paint, an unoccluded window or reading.
+Message and native-item references use the same bounded, single-line opaque ID
+contract as the transcript surfaces that supply them. A Host
+`invalid_request` rejects and remembers that candidate for the current call so
+another current reference may be observed without retrying the rejected one.
+Other failures retry after 1 and 2 seconds, then leave the call's
+first-visible result unknown; later text never backfills a manufactured success.
 
 Xianxian uses one layered sitting body or one VP9 clip at a time. Expiring Host
 facts with task, result and event identities select tool work, workspace fetch,

packages/companion/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@clowder-ai/companion",
-  "version": "0.1.0-alpha.22",
+  "version": "0.1.0-alpha.23",
   "description": "The desktop voice body for your Clowder companion",
   "license": "MIT",
   "type": "module",
@@ -27,8 +27,8 @@
     "prepack": "pnpm build"
   },
   "devDependencies": {
-    "@clowder-ai/plugin-contract": "0.1.0-beta.26",
-    "@clowder-ai/plugin-sdk": "0.1.0-beta.22",
+    "@clowder-ai/plugin-contract": "0.1.0-beta.27",
+    "@clowder-ai/plugin-sdk": "0.1.0-beta.23",
     "playwright": "1.61.1"
   },
   "repository": {

packages/companion/scripts/build.mjs

@@ -20,7 +20,7 @@ const manifest = {
   description: { default: 'Talk, think and find your shared history with your companion.', translations: { 'zh-CN': '和猫猫说话、查资料，回到同一段聊天继续工作。' } },
   runtime: { transport: 'builtin' },
   features: [{ id: 'companion', name: '猫猫球', resources: [], capabilities: ['windows.create'], contributions: [{ type: 'desktop-window', id: 'companion' }] }],
-  contributions: [{ type: 'desktop-window', id: 'companion', role: 'companion', bridgeVersion: '1.3.0',
+  contributions: [{ type: 'desktop-window', id: 'companion', role: 'companion', bridgeVersion: '1.4.0',
     surface: { entrypoint: 'renderer/index.html', integrity: `sha256-${createHash('sha256').update(html).digest('base64')}` },
     presentation: { width: 300, height: 270, frame: false, transparent: true, alwaysOnTop: true, skipTaskbar: true } }],
 };

packages/companion/scripts/preview.mjs

@@ -17,7 +17,7 @@ const page = `<!doctype html><html lang="zh-CN"><meta charset="utf-8"><meta name
 <main><iframe title="实际猫猫球界面" src="/index.html?skin=ragdoll-v1"></iframe></main>
 <script>const frame=document.querySelector('iframe');document.querySelector('#disconnect').onclick=()=>frame.contentWindow.postMessage('preview-disconnect',location.origin);function refresh(){frame.style.width='300px';frame.style.height='270px';frame.src='/index.html?skin='+document.querySelector('#skin').value+'&scenario='+document.querySelector('#scenario').value;}document.querySelectorAll('select').forEach(e=>e.onchange=refresh);document.querySelector('#background').onclick=()=>document.querySelector('main').classList.toggle('dark');window.addEventListener('message',e=>{if(e.source!==frame.contentWindow||e.origin!==location.origin)return;if(e.data.kind==='geometry'){frame.style.width=e.data.width+'px';frame.style.height=e.data.height+'px';}if(e.data.kind==='resize'){frame.style.width=e.data.expanded?'380px':'300px';frame.style.height=e.data.expanded?'590px':'270px';}if(e.data.kind==='note')document.querySelector('#note').textContent=e.data.text;});</script></html>`;
 
-export function fixture(skin, failure, pending = true) {
+export function fixture(skin, failure, pending = true, autoTranscript = true) {
   return `<script>
   const listeners = new Set(); const messages = [{id:'preview-history',role:'assistant',text:'这是已保存的聊天。语音进行中也可以继续查看。',name:'砚砚',companionIdentity:{
     v:1,name:'猫猫球',partner:{catId:'fable-5',displayName:'宪宪',skin:'xianxian-codex'},
@@ -45,7 +45,7 @@ export function fixture(skin, failure, pending = true) {
     switch(command.kind){
       case 'state': return identity();
       case 'prepare': phase='ready';return identity();
-      case 'audio.connect': activeMode=command.mode??'duplex';setTimeout(()=>{if(${failure}){emit({kind:'audio',type:'error',callId,code:'carrier_unavailable'});}else{phase='talking';emit({kind:'audio',type:'connected',callId});emit({kind:'audio',type:'transcript',callId,role:'assistant',text:'这是正在说的话，查看记录不会停止语音。',itemId:'preview-live-output',turnId:'preview-live-turn'});}},450);return {kind:'ok'};
+      case 'audio.connect': activeMode=command.mode??'duplex';setTimeout(()=>{if(${failure}){emit({kind:'audio',type:'error',callId,code:'carrier_unavailable'});}else{phase='talking';emit({kind:'audio',type:'connected',callId});if(${autoTranscript})emit({kind:'audio',type:'transcript',callId,role:'assistant',text:'这是正在说的话，查看记录不会停止语音。',itemId:'preview-live-output',turnId:'preview-live-turn'});}},450);return {kind:'ok'};
       case 'stop': phase='idle';activeMode=null;return {kind:'ok'};
       case 'view.layout': { const petWidth=Math.round(120*companionSettings.ballSize/72),petHeight=Math.round(130*companionSettings.ballSize/72);const width=Math.max(petWidth+16,command.width+16),height=command.panel==='none'?petHeight+16:command.height+petHeight+28; parent.postMessage({kind:'geometry',width,height},location.origin); return {kind:'layout',width,height,pet:{x:(width-petWidth)/2,y:height-petHeight-8},panel:{x:8,y:8,width:command.width,height:command.height}}; }
       case 'view.drag':return {kind:'ok'};
@@ -57,6 +57,7 @@ export function fixture(skin, failure, pending = true) {
       case 'companion.disable':note('正式 Host 会结束媒体并停用当前安装实例；预览保持打开。');return {kind:'companion-lifecycle',action:'disable',outcome:'disabled'};
       case 'conversation.read':return {kind:'conversation',threadTitle:'猫猫球 · 伴随对话',messages,hasMore:false};
       case 'transcript.read':return {kind:'transcript',scope:{callId,realtimeSessionId},rows:transcriptRows,hasMore:false};
+      case 'presentation.observe':return {kind:'ok'};
       case 'decisions.read':return ${pending}?{kind:'decisions',version:1,status:'partial',observedAt:Date.now(),
         sources:{approvals:{status:'available',coverage:'complete'},needsMe:{status:'unavailable',coverage:'unknown'}},
         items:[
@@ -99,7 +100,7 @@ export function createPreviewServer() {
       if (url.pathname === '/index.html') {
         const skin = url.searchParams.get('skin');
         if (!skins.includes(skin)) { res.writeHead(400); res.end('unknown skin'); return; }
-        content = content.toString().replace('<script type="module"', `${fixture(skin, url.searchParams.get('scenario') === 'failure', url.searchParams.get('pending') !== '0')}<script type="module"`);
+        content = content.toString().replace('<script type="module"', `${fixture(skin, url.searchParams.get('scenario') === 'failure', url.searchParams.get('pending') !== '0', url.searchParams.get('autoTranscript') !== '0')}<script type="module"`);
       }
       res.writeHead(200, { 'Content-Type': previewMime[extname(file)], 'Cache-Control': 'no-store' }); res.end(content);
     } catch { res.writeHead(404); res.end(); }

packages/companion/src/call-transcript-view.mjs

@@ -191,9 +191,11 @@ export class CallTranscriptView {
       this.container.scrollTop = this.container.scrollHeight;
       this.setUnread(0);
     } else if (isNew) this.setUnread(this.unread + 1);
-    return true;
+    return row;
   }
 
+  rowForMessage(messageId) { return this.durableRows.get(messageId); }
+
   appendTyped({ receipt, text }) {
     if (!this.callId || receipt?.delivery !== 'accepted' || receipt.callId !== this.callId) return false;
     const pinned = this.nearBottom();

packages/companion/src/recent-bubble.mjs

@@ -1,9 +1,16 @@
 /** A compact projection of Host history plus the current unsaved voice caption. */
+function referenceKey(reference) {
+  if (reference?.kind === 'message') return `message:${reference.messageId}`;
+  if (reference?.kind === 'native-item') return `native-item:${reference.nativeItemId}`;
+  return null;
+}
+
 export class RecentBubble {
   constructor(rows) {
     this.rows = rows;
     this.history = [];
     this.live = new Map();
+    this.rowReferences = new WeakMap();
   }
   load(messages) {
     this.history = messages.slice(-2);
@@ -14,13 +21,16 @@ export class RecentBubble {
       role: row.role,
       text: row.text,
       name: row.source?.kind === 'typed' ? '你' : row.role === 'assistant' ? companionName : '语音',
+      reference: { kind: 'message', messageId: row.messageId },
     })));
   }
-  append(role, text, name = role === 'user' ? '你' : '猫猫') {
+  append(role, text, name = role === 'user' ? '你' : '猫猫', reference) {
     if (!['user', 'assistant'].includes(role) || typeof text !== 'string') return;
     const previous = this.live.get(role);
-    this.live.set(role, { name, text: `${previous?.text ?? ''}${text}`.slice(-240) });
+    this.live.set(role, { name, text: `${previous?.text ?? ''}${text}`.slice(-240),
+      reference: reference ?? previous?.reference });
     this.render();
+    return reference ? this.rowsForReference(reference)[0] : this.rows.find(row => row.dataset?.role === role);
   }
   finish(role) {
     this.live.delete(role);
@@ -38,6 +48,10 @@ export class RecentBubble {
   hasContent() {
     return this.history.length > 0 || this.live.size > 0;
   }
+  rowsForReference(reference) {
+    const key = referenceKey(reference);
+    return key ? this.rows.filter(row => this.rowReferences.get(row) === key) : [];
+  }
   render() {
     const live = [...this.live].map(([role, value]) => ({ role, ...value }));
     const entries = [...this.history.slice(-Math.max(0, 2 - live.length)), ...live].slice(-2);
@@ -69,6 +83,9 @@ export class RecentBubble {
       }
       row.dataset ??= {};
       row.dataset.role = entry?.role ?? '';
+      const key = referenceKey(entry?.reference);
+      if (key) this.rowReferences.set(row, key);
+      else this.rowReferences.delete(row);
     }
   }
 }

packages/companion/src/surface.mjs

@@ -19,6 +19,7 @@ import { FoldedChatUnread } from './chat-unread.mjs';
 import { COMPACT_BALL_SIZE, compactSizeAction } from './compact-size.mjs';
 import { WorkOverviewView } from './work-overview.mjs';
 import { MeetingStatus, renderMeetingStatus } from './meeting-state.mjs';
+import { VisibleTextPresentation } from './visible-text-receipt.mjs';
 const $ = id => document.getElementById(id);
 const label = (id, text) => { $(id).querySelector('.label').textContent = text; };
 const status = text => {
@@ -62,6 +63,11 @@ if (!window.clowderCompanion) {
 } else {
   nativeTravel.start();
   const client = createCompanionClient(window.clowderCompanion);
+  const visibleText = new VisibleTextPresentation({
+    document,
+    report: observation => client.observePresentation(observation),
+    transcript, callTranscript, bubble, callBubble,
+  });
   const controls = bindPetControls(client, {
     error: (error, context) => status(context?.panel === 'settings' && error?.code === 'invalid_request' ? '这个 Host 版本还不支持猫旁设置。原来的设置入口会继续保留。' : explainError(error)),
     moved: (dx, dy) => {
@@ -74,6 +80,7 @@ if (!window.clowderCompanion) {
       setTimeout(() => motion.setDockedEdge(detectDockedEdge(window, $('pet').getBoundingClientRect())), 100);
     },
     changed: panel => {
+      visibleText.retry();
       if (panel === 'chat') foldedChatUnread.open();
       else if (previousControlPanel === 'chat' && conversation?.active) foldedChatUnread.fold();
       previousControlPanel = panel;
@@ -147,10 +154,8 @@ if (!window.clowderCompanion) {
         callTranscript.appendTyped(event);
         void readHistory(); void readCallTranscript();
       } else {
-        transcript.append(event.role, event.text, true);
-        callTranscript.appendLive(event);
         const speaker = event.role === 'assistant' ? conversation.identity?.displayName ?? '猫猫' : '语音';
-        bubble.append(event.role, event.text, speaker); callBubble.append(event.role, event.text, speaker);
+        visibleText.appendLive(event, speaker);
         void controls.setAmbient('actions');
       }
     },
@@ -167,7 +172,7 @@ if (!window.clowderCompanion) {
       const receiveOnly = value.audioMode === 'receive_only';
       const canReceiveOnly = value.identity?.audio?.supportedModes?.includes('receive_only') === true;
       const retryReceiveOnly = value.failed && value.retryMode === 'receive_only' && canReceiveOnly;
-      callTranscript.setCall(value.callId);
+      visibleText.setCall(value.callId);
       void controls.setAmbient(ambientPanel(active));
       $('begin').hidden = active; $('begin').disabled = !value.identity;
       $('listen').hidden = active || !canReceiveOnly; $('listen').disabled = !value.identity;
@@ -305,9 +310,8 @@ if (!window.clowderCompanion) {
     try {
       const reply = await client.readTranscript();
       if (!conversation.active || conversation.callId !== callId) return;
-      callTranscript.load(reply);
+      if (!visibleText.loadTranscript(callId, reply, conversation.identity?.displayName)) return;
       foldedChatUnread.receiveMany(reply.rows);
-      callBubble.loadTranscript(reply.rows, conversation.identity?.displayName);
       $('call-transcript-status').textContent = reply.hasMore ? '这里只显示这次通话最近的内容' : '';
     } catch {
       if (conversation.active && conversation.callId === callId) $('call-transcript-status').textContent = '字幕暂未更新 · 实时内容仍会继续显示';
@@ -340,6 +344,6 @@ if (!window.clowderCompanion) {
   void conversation.refresh(); void readHistory(); void decisionPanel.read(); void settingsView.controller.load(); void controls.show('none');
   window.addEventListener('beforeunload', () => {
     clearInterval(statusMonitor); clearInterval(historyMonitor); clearInterval(transcriptMonitor); clearInterval(decisionMonitor); clearInterval(workMonitor);
-    unsubscribe(); nativeTravel.close(); motion.close(); void conversation.end();
+    unsubscribe(); visibleText.close(); nativeTravel.close(); motion.close(); void conversation.end();
   });
 }

packages/companion/src/transcript-view.mjs

@@ -20,6 +20,7 @@ export class TranscriptView {
     }
     row.textContent = (row.textContent + text).slice(-16000);
     if (pinned) this.container.scrollTop = this.container.scrollHeight;
+    return row;
   }
   finish({ role, transcript, turnId }) {
     if (!['user', 'assistant'].includes(role) || (turnId && this.completed.has(turnId))) return;