feat(agent_meetings): wire mascotId through full stack + fix tests

app/src-tauri/src/meet_video/camera_bridge.js

@@ -9,7 +9,7 @@
 //
 // Primary: connect to `ws://127.0.0.1:<frameBusPort>` (Rust-hosted, see
 // `frame_bus.rs`) and pump incoming binary JPEG frames straight onto
-// our 640×480 capture canvas. This is what the user sees in Meet.
+// our 1280x720 capture canvas. This is what the user sees in Meet.
 //
 // Fallback: if the WS hasn't delivered a frame in the last 500 ms (or
 // the port is 0 — meaning the producer never came up), draw the
@@ -23,8 +23,8 @@
 (function () {
   if (window.__openhumanCameraBridge) return;
   const TAG = '[openhuman-camera-bridge]';
-  const W = 640;
-  const H = 480;
+  const W = 1280;
+  const H = 720;
   const FPS = 30;
   const FRAME_BUS_PORT = __OPENHUMAN_FRAME_BUS_PORT__;
   // The static-SVG path is **cold-start only**: we use it before the
@@ -63,6 +63,12 @@
   let nextRecvSeq = 0;
   let lastAcceptedSeq = -1;
   let wsState = 'init';
+  let lastRemoteBitmapInfo = null;
+  let lastDrawSource = 'cold-start';
+  let lastCanvasProbe = null;
+  let lastOutboundVideoStats = null;
+  let lastOutboundStatsAt = 0;
+  const peerConnections = new Set();
 
   function loadImage(src) {
     return new Promise(function (resolve, reject) {
@@ -145,6 +151,12 @@
         }
         latestRemoteBitmap = bitmap;
         latestRemoteAt = Date.now();
+        lastRemoteBitmapInfo = {
+          width: bitmap.width || null,
+          height: bitmap.height || null,
+          bytes: ev.data.byteLength || 0,
+          seq: mySeq,
+        };
         lastAcceptedSeq = mySeq;
         remoteFrameCount++;
       } catch (err) {
@@ -170,6 +182,45 @@
   // setInterval keeps firing regardless of focus, which is what we need
   // for the outbound camera to stay live.
   let frame = 0;
+  function sampleCanvasPixels() {
+    try {
+      const cols = 7;
+      const rows = 5;
+      let sum = 0;
+      let min = 255;
+      let max = 0;
+      let count = 0;
+      let dark = 0;
+      let bright = 0;
+      for (let y = 0; y < rows; y++) {
+        for (let x = 0; x < cols; x++) {
+          const px = Math.max(0, Math.min(W - 1, Math.floor(((x + 0.5) * W) / cols)));
+          const py = Math.max(0, Math.min(H - 1, Math.floor(((y + 0.5) * H) / rows)));
+          const d = ctx.getImageData(px, py, 1, 1).data;
+          const luma = Math.round((d[0] * 0.299) + (d[1] * 0.587) + (d[2] * 0.114));
+          sum += luma;
+          min = Math.min(min, luma);
+          max = Math.max(max, luma);
+          if (luma < 8) dark++;
+          if (luma > 32) bright++;
+          count++;
+        }
+      }
+      lastCanvasProbe = {
+        avgLuma: Math.round(sum / Math.max(1, count)),
+        minLuma: min,
+        maxLuma: max,
+        darkSamples: dark,
+        brightSamples: bright,
+        sampleCount: count,
+        source: lastDrawSource,
+        frame: frame,
+      };
+    } catch (err) {
+      lastCanvasProbe = { error: String((err && err.message) || err), source: lastDrawSource };
+    }
+  }
+
   function tick() {
     frame++;
     if (latestRemoteBitmap) {
@@ -189,6 +240,8 @@
       const dx = (W - dw) / 2;
       const dy = (H - dh) / 2 + (Math.sin(frame / (FPS * 2 / Math.PI)) * 0.5);
       ctx.drawImage(latestRemoteBitmap, dx, dy, dw, dh);
+      lastDrawSource = 'remote';
+      if (frame % FPS === 0) sampleCanvasPixels();
       return;
     }
     // Cold-start fallback: static SVG with a gentle bob so the camera
@@ -208,6 +261,8 @@
       const dy = (H - dh) / 2 + bob;
       ctx.drawImage(img, dx, dy, dw, dh);
     }
+    lastDrawSource = 'fallback';
+    if (frame % FPS === 0) sampleCanvasPixels();
   }
   setInterval(tick, Math.round(1000 / FPS));
 
@@ -220,6 +275,9 @@
         configurable: true,
       });
     } catch (_) {}
+    try {
+      fakeVideoTrack.contentHint = 'motion';
+    } catch (_) {}
   }
 
   // ---- monkey-patch ----------------------------------------------------
@@ -252,6 +310,149 @@
     return v === true || (v && typeof v === 'object');
   }
 
+  function makeMascotTrack() {
+    const ours = stream.getVideoTracks()[0];
+    if (!ours) return null;
+    const clone = ours.clone();
+    try {
+      Object.defineProperty(clone, 'label', {
+        value: 'OpenHuman Mascot',
+        configurable: true,
+      });
+    } catch (_) {}
+    try {
+      clone.contentHint = 'motion';
+    } catch (_) {}
+    return clone;
+  }
+
+  function isVideoTrack(track) {
+    return !!track && track.kind === 'video';
+  }
+
+  function isVideoTransceiverInit(init) {
+    if (!init || typeof init !== 'object') return false;
+    if (Array.isArray(init.streams) && init.streams.some(function (s) {
+      return s && typeof s.getVideoTracks === 'function' && s.getVideoTracks().length > 0;
+    })) return true;
+    return false;
+  }
+
+  function sanitizeVideoSenderInit(init) {
+    if (!init || typeof init !== 'object' || !Array.isArray(init.sendEncodings)) return init;
+    if (init.sendEncodings.length <= 1) return init;
+    const next = Object.assign({}, init);
+    const first = Object.assign({}, init.sendEncodings[0] || {});
+    delete first.rid;
+    delete first.scalabilityMode;
+    first.scaleResolutionDownBy = 1;
+    next.sendEncodings = [first];
+    console.log(TAG, 'collapsed video sendEncodings to one layer for mascot');
+    return next;
+  }
+
+  async function collectOutboundVideoStats() {
+    const now = Date.now();
+    if (now - lastOutboundStatsAt < 2000) return;
+    lastOutboundStatsAt = now;
+    try {
+      for (const pc of peerConnections) {
+        if (!pc || typeof pc.getSenders !== 'function') continue;
+        const senders = pc.getSenders().filter(function (sender) {
+          return sender && sender.track && sender.track.kind === 'video';
+        });
+        for (const sender of senders) {
+          if (typeof sender.getStats !== 'function') continue;
+          const report = await sender.getStats();
+          report.forEach(function (stat) {
+            if (stat && stat.type === 'outbound-rtp' && (stat.kind === 'video' || stat.mediaType === 'video')) {
+              lastOutboundVideoStats = {
+                framesEncoded: stat.framesEncoded ?? null,
+                framesSent: stat.framesSent ?? null,
+                bytesSent: stat.bytesSent ?? null,
+                frameWidth: stat.frameWidth ?? null,
+                frameHeight: stat.frameHeight ?? null,
+                qualityLimitationReason: stat.qualityLimitationReason ?? null,
+                timestamp: Math.round(stat.timestamp || 0),
+              };
+            }
+          });
+        }
+      }
+    } catch (err) {
+      lastOutboundVideoStats = { error: String((err && err.message) || err) };
+    }
+  }
+
+  function sampleVideoLuma(video) {
+    try {
+      if (!video || !video.videoWidth || !video.videoHeight || video.readyState < 2) return null;
+      const c = document.createElement('canvas');
+      c.width = 16;
+      c.height = 9;
+      const cctx = c.getContext('2d', { alpha: false });
+      if (!cctx) return null;
+      cctx.drawImage(video, 0, 0, c.width, c.height);
+      const data = cctx.getImageData(0, 0, c.width, c.height).data;
+      let sum = 0;
+      let min = 255;
+      let max = 0;
+      for (let i = 0; i < data.length; i += 4) {
+        const luma = Math.round((data[i] * 0.299) + (data[i + 1] * 0.587) + (data[i + 2] * 0.114));
+        sum += luma;
+        min = Math.min(min, luma);
+        max = Math.max(max, luma);
+      }
+      const count = data.length / 4;
+      return { avgLuma: Math.round(sum / Math.max(1, count)), minLuma: min, maxLuma: max };
+    } catch (err) {
+      return { error: String((err && err.message) || err) };
+    }
+  }
+
+  function probeVideoElements() {
+    try {
+      return Array.prototype.slice.call(document.querySelectorAll('video'), 0, 12).map(function (video, idx) {
+        const rect = video.getBoundingClientRect ? video.getBoundingClientRect() : null;
+        const tracks = video.srcObject && typeof video.srcObject.getVideoTracks === 'function'
+          ? video.srcObject.getVideoTracks().map(function (track) {
+              let settings = {};
+              try { settings = track.getSettings ? track.getSettings() : {}; } catch (_) {}
+              return {
+                // track.label is intentionally omitted — on real devices it
+                // contains the camera/microphone device name, which is PII.
+                enabled: !!track.enabled,
+                muted: !!track.muted,
+                readyState: track.readyState || '',
+                width: settings.width || null,
+                height: settings.height || null,
+                frameRate: settings.frameRate || null,
+              };
+            })
+          : [];
+        return {
+          idx: idx,
+          videoWidth: video.videoWidth || 0,
+          videoHeight: video.videoHeight || 0,
+          readyState: video.readyState,
+          paused: !!video.paused,
+          currentTime: Math.round((video.currentTime || 0) * 1000) / 1000,
+          visible: !!rect && rect.width > 0 && rect.height > 0,
+          rect: rect ? {
+            width: Math.round(rect.width),
+            height: Math.round(rect.height),
+            x: Math.round(rect.x),
+            y: Math.round(rect.y),
+          } : null,
+          tracks: tracks,
+          luma: sampleVideoLuma(video),
+        };
+      });
+    } catch (err) {
+      return [{ error: String((err && err.message) || err) }];
+    }
+  }
+
   md.getUserMedia = async function (constraints) {
     console.log(TAG, 'getUserMedia intercepted', JSON.stringify(constraints || {}));
     if (!wantsVideo(constraints)) {
@@ -269,13 +470,86 @@
     }
     const ours = stream.getVideoTracks()[0];
     if (ours) {
-      realStream.addTrack(ours.clone());
+      realStream.addTrack(makeMascotTrack());
     } else {
       console.warn(TAG, 'no canvas video track available — returning audio-only');
     }
     return realStream;
   };
 
+  const NativeRTCPeerConnection = window.RTCPeerConnection || window.webkitRTCPeerConnection;
+  if (NativeRTCPeerConnection && !NativeRTCPeerConnection.__openhumanCameraPatched) {
+    const origAddTrack = NativeRTCPeerConnection.prototype.addTrack;
+    const origAddTransceiver = NativeRTCPeerConnection.prototype.addTransceiver;
+    const origGetSenders = NativeRTCPeerConnection.prototype.getSenders;
+
+    if (origAddTrack) {
+      NativeRTCPeerConnection.prototype.addTrack = function (track) {
+        peerConnections.add(this);
+        const args = Array.prototype.slice.call(arguments);
+        if (isVideoTrack(track)) {
+          const mascot = makeMascotTrack();
+          if (mascot) {
+            args[0] = mascot;
+            console.log(TAG, 'RTCPeerConnection.addTrack video -> mascot');
+          }
+        }
+        return origAddTrack.apply(this, args);
+      };
+    }
+
+    if (origAddTransceiver) {
+      NativeRTCPeerConnection.prototype.addTransceiver = function (trackOrKind, init) {
+        peerConnections.add(this);
+        let nextTrackOrKind = trackOrKind;
+        let nextInit = init;
+        const direction = init && init.direction;
+        const willSend = !direction || direction === 'sendrecv' || direction === 'sendonly';
+        if (willSend && (isVideoTrack(trackOrKind) || isVideoTransceiverInit(init))) {
+          const mascot = makeMascotTrack();
+          if (mascot) {
+            nextTrackOrKind = mascot;
+            nextInit = sanitizeVideoSenderInit(init);
+            console.log(TAG, 'RTCPeerConnection.addTransceiver video -> mascot');
+          }
+        }
+        return origAddTransceiver.call(this, nextTrackOrKind, nextInit);
+      };
+    }
+
+    if (origGetSenders) {
+      NativeRTCPeerConnection.prototype.getSenders = function () {
+        peerConnections.add(this);
+        return origGetSenders.apply(this, arguments);
+      };
+    }
+
+    NativeRTCPeerConnection.__openhumanCameraPatched = true;
+  }
+
+  if (window.RTCRtpSender && window.RTCRtpSender.prototype && window.RTCRtpSender.prototype.replaceTrack) {
+    const origReplaceTrack = window.RTCRtpSender.prototype.replaceTrack;
+    if (!origReplaceTrack.__openhumanCameraPatched) {
+      const patchedReplaceTrack = function (track) {
+        const args = Array.prototype.slice.call(arguments);
+        if (isVideoTrack(track)) {
+          const mascot = makeMascotTrack();
+          if (mascot) {
+            args[0] = mascot;
+            console.log(TAG, 'RTCRtpSender.replaceTrack video -> mascot');
+          }
+        }
+        return origReplaceTrack.apply(this, args);
+      };
+      patchedReplaceTrack.__openhumanCameraPatched = true;
+      window.RTCRtpSender.prototype.replaceTrack = patchedReplaceTrack;
+    }
+  }
+
+  setInterval(function () {
+    void collectOutboundVideoStats();
+  }, 2000);
+
   // ---- host API --------------------------------------------------------
   window.__openhumanSetMood = function (mood) {
     if (!Object.prototype.hasOwnProperty.call(MASCOTS, mood)) {
@@ -300,6 +574,18 @@
       remoteFrameCount: remoteFrameCount,
       droppedOutOfOrder: droppedOutOfOrder,
       remoteFreshMs: latestRemoteAt ? (Date.now() - latestRemoteAt) : null,
+      lastRemoteBitmapInfo: lastRemoteBitmapInfo,
+      lastDrawSource: lastDrawSource,
+      canvasProbe: lastCanvasProbe,
+      outboundVideoStats: lastOutboundVideoStats,
+      videoTrack: fakeVideoTrack ? {
+        label: fakeVideoTrack.label,
+        enabled: fakeVideoTrack.enabled,
+        muted: fakeVideoTrack.muted,
+        readyState: fakeVideoTrack.readyState,
+        settings: fakeVideoTrack.getSettings ? fakeVideoTrack.getSettings() : null,
+      } : null,
+      videoElements: probeVideoElements(),
     };
   };