diff --git a/Dockerfile b/Dockerfile index 5e724237..4d65eb22 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,7 @@ COPY packages/gui/package.json packages/gui/ COPY packages/a2a/package.json packages/a2a/ COPY packages/cli/package.json packages/cli/ COPY packages/web-ui/package.json packages/web-ui/ +COPY packages/chrome-extension/package.json packages/chrome-extension/ RUN pnpm install --frozen-lockfile || pnpm install # ── Stage 2: Build all packages ───────────────────────────────────────────── diff --git a/package.json b/package.json index 12876e7d..0e4ab0df 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,8 @@ }, "pnpm": { "onlyBuiltDependencies": [ - "esbuild" + "esbuild", + "node-datachannel" ] } } diff --git a/packages/chrome-extension/icons/icon128.png b/packages/chrome-extension/icons/icon128.png new file mode 100644 index 00000000..95c6cac6 Binary files /dev/null and b/packages/chrome-extension/icons/icon128.png differ diff --git a/packages/chrome-extension/icons/icon16.png b/packages/chrome-extension/icons/icon16.png new file mode 100644 index 00000000..2de67583 Binary files /dev/null and b/packages/chrome-extension/icons/icon16.png differ diff --git a/packages/chrome-extension/icons/icon48.png b/packages/chrome-extension/icons/icon48.png new file mode 100644 index 00000000..5449da3c Binary files /dev/null and b/packages/chrome-extension/icons/icon48.png differ diff --git a/packages/chrome-extension/manifest.json b/packages/chrome-extension/manifest.json new file mode 100644 index 00000000..4299c697 --- /dev/null +++ b/packages/chrome-extension/manifest.json @@ -0,0 +1,25 @@ +{ + "manifest_version": 3, + "name": "Markus Browser Automation", + "description": "Enables Markus AI agents to automate browser tasks without the remote debugging dialog.", + "version": "1.0.0", + "permissions": ["debugger", "tabs", "activeTab", "scripting"], + "host_permissions": [""], + "background": { + "service_worker": "dist/background.js", + "type": "module" + }, + "icons": { + "16": "icons/icon16.png", + "48": "icons/icon48.png", + "128": "icons/icon128.png" + }, + "action": { + "default_icon": { + "16": "icons/icon16.png", + "48": "icons/icon48.png" + }, + "default_title": "Markus Browser Automation", + "default_popup": "popup.html" + } +} diff --git a/packages/chrome-extension/pack.mjs b/packages/chrome-extension/pack.mjs new file mode 100644 index 00000000..94760190 --- /dev/null +++ b/packages/chrome-extension/pack.mjs @@ -0,0 +1,138 @@ +/** + * Pack the Chrome extension into a zip file ready for distribution. + * Includes only the files needed to load the extension in Chrome. + * + * Output: dist/markus-browser-extension.zip + */ +import { createWriteStream, readFileSync, readdirSync, statSync, existsSync } from 'node:fs'; +import { join, relative } from 'node:path'; +import { createDeflateRaw } from 'node:zlib'; + +const ROOT = new URL('.', import.meta.url).pathname.replace(/\/$/, ''); +const OUT = join(ROOT, 'dist', 'markus-browser-extension.zip'); + +const FILES = [ + 'manifest.json', + 'popup.html', + 'popup.js', + 'dist/background.js', + 'icons/icon16.png', + 'icons/icon48.png', + 'icons/icon128.png', +]; + +// Verify all files exist +for (const f of FILES) { + if (!existsSync(join(ROOT, f))) { + console.error(`Missing file: ${f}. Run "pnpm run build" first.`); + process.exit(1); + } +} + +// Minimal zip writer (no external deps) +class ZipWriter { + constructor(outPath) { + this.entries = []; + this.stream = createWriteStream(outPath); + this.offset = 0; + } + + async addFile(archivePath, content) { + const header = Buffer.alloc(30); + const nameBytes = Buffer.from(archivePath, 'utf8'); + const compressed = await this._deflate(content); + const crc = this._crc32(content); + + // Local file header + header.writeUInt32LE(0x04034b50, 0); // signature + header.writeUInt16LE(20, 4); // version needed + header.writeUInt16LE(0, 6); // flags + header.writeUInt16LE(8, 8); // compression: deflate + header.writeUInt16LE(0, 10); // mod time + header.writeUInt16LE(0, 12); // mod date + header.writeUInt32LE(crc, 14); // crc-32 + header.writeUInt32LE(compressed.length, 18); // compressed size + header.writeUInt32LE(content.length, 22); // uncompressed size + header.writeUInt16LE(nameBytes.length, 26); // filename length + header.writeUInt16LE(0, 28); // extra field length + + const localOffset = this.offset; + this._write(header); + this._write(nameBytes); + this._write(compressed); + + this.entries.push({ archivePath, nameBytes, crc, compressedSize: compressed.length, uncompressedSize: content.length, localOffset }); + } + + finish() { + const cdStart = this.offset; + for (const e of this.entries) { + const cdh = Buffer.alloc(46); + cdh.writeUInt32LE(0x02014b50, 0); // signature + cdh.writeUInt16LE(20, 4); // version made by + cdh.writeUInt16LE(20, 6); // version needed + cdh.writeUInt16LE(0, 8); // flags + cdh.writeUInt16LE(8, 10); // compression + cdh.writeUInt16LE(0, 12); // time + cdh.writeUInt16LE(0, 14); // date + cdh.writeUInt32LE(e.crc, 16); + cdh.writeUInt32LE(e.compressedSize, 20); + cdh.writeUInt32LE(e.uncompressedSize, 24); + cdh.writeUInt16LE(e.nameBytes.length, 28); + cdh.writeUInt16LE(0, 30); // extra + cdh.writeUInt16LE(0, 32); // comment + cdh.writeUInt16LE(0, 34); // disk + cdh.writeUInt16LE(0, 36); // internal attrs + cdh.writeUInt32LE(0, 38); // external attrs + cdh.writeUInt32LE(e.localOffset, 42); + this._write(cdh); + this._write(e.nameBytes); + } + const cdSize = this.offset - cdStart; + + // End of central directory + const eocd = Buffer.alloc(22); + eocd.writeUInt32LE(0x06054b50, 0); + eocd.writeUInt16LE(this.entries.length, 8); + eocd.writeUInt16LE(this.entries.length, 10); + eocd.writeUInt32LE(cdSize, 12); + eocd.writeUInt32LE(cdStart, 16); + this._write(eocd); + this.stream.end(); + } + + _write(buf) { + this.stream.write(buf); + this.offset += buf.length; + } + + _deflate(data) { + return new Promise((resolve, reject) => { + const chunks = []; + const deflater = createDeflateRaw(); + deflater.on('data', c => chunks.push(c)); + deflater.on('end', () => resolve(Buffer.concat(chunks))); + deflater.on('error', reject); + deflater.end(data); + }); + } + + _crc32(buf) { + let crc = 0xFFFFFFFF; + for (let i = 0; i < buf.length; i++) { + crc ^= buf[i]; + for (let j = 0; j < 8; j++) crc = (crc >>> 1) ^ (crc & 1 ? 0xEDB88320 : 0); + } + return (crc ^ 0xFFFFFFFF) >>> 0; + } +} + +const zip = new ZipWriter(OUT); +for (const f of FILES) { + const content = readFileSync(join(ROOT, f)); + await zip.addFile(f, content); +} +zip.finish(); + +const size = statSync(OUT).size; +console.log(`Packed ${FILES.length} files → ${OUT} (${(size / 1024).toFixed(1)} KB)`); diff --git a/packages/chrome-extension/package.json b/packages/chrome-extension/package.json new file mode 100644 index 00000000..983c0bdc --- /dev/null +++ b/packages/chrome-extension/package.json @@ -0,0 +1,16 @@ +{ + "name": "@markus/chrome-extension", + "version": "1.0.0", + "private": true, + "scripts": { + "build": "esbuild src/background.ts --bundle --outfile=dist/background.js --format=esm --target=es2022 --platform=browser", + "watch": "esbuild src/background.ts --bundle --outfile=dist/background.js --format=esm --target=es2022 --platform=browser --watch", + "pack": "pnpm run build && node pack.mjs", + "prepare": "pnpm run build" + }, + "devDependencies": { + "@types/chrome": "^0.0.300", + "esbuild": "^0.25.0", + "typescript": "^5.6.0" + } +} diff --git a/packages/chrome-extension/popup.html b/packages/chrome-extension/popup.html new file mode 100644 index 00000000..61521d18 --- /dev/null +++ b/packages/chrome-extension/popup.html @@ -0,0 +1,84 @@ + + + + + + +
+ +

Markus Browser

+
+
+ + Checking... +
+
+
+ Bridge + ws://127.0.0.1:9333 +
+
+ Pages tracked + 0 +
+
+
+
+ Enables Markus agents to automate Chrome without the remote debugging dialog. +
+ + + + diff --git a/packages/chrome-extension/popup.js b/packages/chrome-extension/popup.js new file mode 100644 index 00000000..da49d591 --- /dev/null +++ b/packages/chrome-extension/popup.js @@ -0,0 +1,25 @@ +// Query the background service worker for connection status +chrome.runtime.sendMessage({ type: 'getStatus' }, (response) => { + if (chrome.runtime.lastError || !response) { + document.getElementById('statusText').textContent = 'Extension loading...'; + return; + } + + const statusEl = document.getElementById('status'); + const dotEl = document.getElementById('dot'); + const textEl = document.getElementById('statusText'); + const pageCountEl = document.getElementById('pageCount'); + + if (response.connected) { + statusEl.className = 'status connected'; + dotEl.className = 'dot green'; + textEl.textContent = 'Connected to Markus'; + } else { + statusEl.className = 'status disconnected'; + dotEl.className = 'dot gray'; + textEl.textContent = 'Not connected'; + } + + pageCountEl.textContent = String(response.pageCount || 0); + document.getElementById('bridgeUrl').textContent = response.bridgeUrl || 'ws://127.0.0.1:9333'; +}); diff --git a/packages/chrome-extension/src/background.ts b/packages/chrome-extension/src/background.ts new file mode 100644 index 00000000..8694a31e --- /dev/null +++ b/packages/chrome-extension/src/background.ts @@ -0,0 +1,56 @@ +/** + * Chrome Extension Service Worker — main entry point. + * + * Connects to Markus browser bridge via WebSocket and registers + * all tool handlers that mirror chrome-devtools-mcp's API. + */ + +import { BridgeClient } from './protocol.js'; +import { PageManager } from './page-manager.js'; +import { registerNavigationTools } from './tools/navigation.js'; +import { registerInputTools } from './tools/input.js'; +import { registerInspectionTools, setupConsoleListener } from './tools/inspection.js'; +import { registerNetworkTools, setupNetworkListener } from './tools/network.js'; + +const pm = new PageManager(); +const client = new BridgeClient(); + +// Register all tool handlers +registerNavigationTools((name, handler) => client.registerHandler(name, handler), pm); +registerInputTools((name, handler) => client.registerHandler(name, handler), pm); +registerInspectionTools((name, handler) => client.registerHandler(name, handler), pm); +registerNetworkTools((name, handler) => client.registerHandler(name, handler), pm); + +// Set up CDP event listeners +setupConsoleListener(); +setupNetworkListener(); + +// Clean up page state when tabs are closed +chrome.tabs.onRemoved.addListener((tabId) => { + pm.removeByTabId(tabId); + client.send({ event: 'tab_closed', data: { tabId } }); +}); + +// Handle debugger detach events +chrome.debugger.onDetach.addListener((source) => { + if (source.tabId) { + pm.setDebuggerAttached(source.tabId, false); + } +}); + +// Handle popup status queries +chrome.runtime.onMessage.addListener((msg, _sender, sendResponse) => { + if (msg.type === 'getStatus') { + sendResponse({ + connected: client.connected, + pageCount: pm.getAllPages().length, + bridgeUrl: 'ws://127.0.0.1:9333', + }); + return true; + } +}); + +// Connect to bridge +client.connect(); + +console.log('[Markus] Browser automation extension initialized'); diff --git a/packages/chrome-extension/src/page-manager.ts b/packages/chrome-extension/src/page-manager.ts new file mode 100644 index 00000000..7257afc8 --- /dev/null +++ b/packages/chrome-extension/src/page-manager.ts @@ -0,0 +1,82 @@ +/** + * Manages the mapping between sequential page IDs (1, 2, 3...) + * used by the MCP protocol and Chrome's native tab IDs. + * Also tracks which page is "selected" (active for CDP operations). + */ + +export class PageManager { + private tabToPage = new Map(); + private pageToTab = new Map(); + private nextPageId = 1; + private _selectedPageId: number | null = null; + private debuggerAttached = new Set(); + + get selectedPageId(): number | null { return this._selectedPageId; } + get selectedTabId(): number | null { + if (this._selectedPageId === null) return null; + return this.pageToTab.get(this._selectedPageId) ?? null; + } + + getPageId(tabId: number): number { + let pageId = this.tabToPage.get(tabId); + if (pageId === undefined) { + pageId = this.nextPageId++; + this.tabToPage.set(tabId, pageId); + this.pageToTab.set(pageId, tabId); + } + return pageId; + } + + getTabId(pageId: number): number | undefined { + return this.pageToTab.get(pageId); + } + + selectPage(pageId: number): boolean { + if (!this.pageToTab.has(pageId)) return false; + this._selectedPageId = pageId; + return true; + } + + removePage(pageId: number): void { + const tabId = this.pageToTab.get(pageId); + if (tabId !== undefined) { + this.tabToPage.delete(tabId); + this.debuggerAttached.delete(tabId); + } + this.pageToTab.delete(pageId); + if (this._selectedPageId === pageId) { + this._selectedPageId = null; + } + } + + removeByTabId(tabId: number): void { + const pageId = this.tabToPage.get(tabId); + if (pageId !== undefined) { + this.removePage(pageId); + } + } + + isDebuggerAttached(tabId: number): boolean { + return this.debuggerAttached.has(tabId); + } + + setDebuggerAttached(tabId: number, attached: boolean): void { + if (attached) { + this.debuggerAttached.add(tabId); + } else { + this.debuggerAttached.delete(tabId); + } + } + + getAllPages(): Array<{ pageId: number; tabId: number }> { + return [...this.pageToTab.entries()].map(([pageId, tabId]) => ({ pageId, tabId })); + } + + clear(): void { + this.tabToPage.clear(); + this.pageToTab.clear(); + this.debuggerAttached.clear(); + this._selectedPageId = null; + this.nextPageId = 1; + } +} diff --git a/packages/chrome-extension/src/protocol.ts b/packages/chrome-extension/src/protocol.ts new file mode 100644 index 00000000..d40f892b --- /dev/null +++ b/packages/chrome-extension/src/protocol.ts @@ -0,0 +1,151 @@ +/** + * WebSocket client that connects to the Markus browser bridge. + * Handles reconnection, keepalive, and message routing. + */ + +export interface BridgeRequest { + id: number; + method: string; + params: Record; +} + +export interface BridgeResponse { + id: number; + result?: unknown; + error?: string; +} + +export type ToolHandler = (params: Record) => Promise; + +const DEFAULT_URL = 'ws://127.0.0.1:9333'; +const RECONNECT_INTERVAL_MS = 3000; +const KEEPALIVE_INTERVAL_MS = 25000; + +export class BridgeClient { + private ws: WebSocket | null = null; + private url: string; + private handlers = new Map(); + private reconnectTimer: ReturnType | null = null; + private keepaliveTimer: ReturnType | null = null; + private _connected = false; + + constructor(url?: string) { + this.url = url ?? DEFAULT_URL; + } + + get connected(): boolean { return this._connected; } + + registerHandler(method: string, handler: ToolHandler): void { + this.handlers.set(method, handler); + } + + connect(): void { + this.cleanup(); + + try { + this.ws = new WebSocket(this.url); + } catch { + this.scheduleReconnect(); + return; + } + + this.ws.onopen = () => { + console.log('[Markus] Connected to bridge'); + this._connected = true; + this.startKeepalive(); + chrome.action.setIcon({ path: { + '16': 'icons/icon16.png', + '48': 'icons/icon48.png', + }}); + chrome.action.setTitle({ title: 'Markus Browser Automation (Connected)' }); + }; + + this.ws.onclose = () => { + console.log('[Markus] Disconnected from bridge'); + this._connected = false; + this.stopKeepalive(); + chrome.action.setTitle({ title: 'Markus Browser Automation (Disconnected)' }); + this.scheduleReconnect(); + }; + + this.ws.onerror = () => { + // onclose will fire after this + }; + + this.ws.onmessage = (event) => { + try { + const msg = JSON.parse(event.data as string) as BridgeRequest; + this.handleRequest(msg); + } catch (err) { + console.error('[Markus] Failed to parse message:', err); + } + }; + } + + private async handleRequest(req: BridgeRequest): Promise { + const handler = this.handlers.get(req.method); + if (!handler) { + this.send({ id: req.id, error: `Unknown method: ${req.method}` }); + return; + } + + try { + const result = await handler(req.params); + this.send({ id: req.id, result }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + this.send({ id: req.id, error: msg }); + } + } + + send(msg: BridgeResponse | { event: string; data: unknown }): void { + if (this.ws?.readyState === WebSocket.OPEN) { + this.ws.send(JSON.stringify(msg)); + } + } + + private scheduleReconnect(): void { + if (this.reconnectTimer) return; + this.reconnectTimer = setTimeout(() => { + this.reconnectTimer = null; + this.connect(); + }, RECONNECT_INTERVAL_MS); + } + + private startKeepalive(): void { + this.stopKeepalive(); + this.keepaliveTimer = setInterval(() => { + if (this.ws?.readyState === WebSocket.OPEN) { + this.send({ event: 'keepalive', data: { timestamp: Date.now() } }); + } + }, KEEPALIVE_INTERVAL_MS); + } + + private stopKeepalive(): void { + if (this.keepaliveTimer) { + clearInterval(this.keepaliveTimer); + this.keepaliveTimer = null; + } + } + + private cleanup(): void { + this.stopKeepalive(); + if (this.reconnectTimer) { + clearTimeout(this.reconnectTimer); + this.reconnectTimer = null; + } + if (this.ws) { + this.ws.onopen = null; + this.ws.onclose = null; + this.ws.onerror = null; + this.ws.onmessage = null; + try { this.ws.close(); } catch { /* ignore */ } + this.ws = null; + } + this._connected = false; + } + + disconnect(): void { + this.cleanup(); + } +} diff --git a/packages/chrome-extension/src/tools/input.ts b/packages/chrome-extension/src/tools/input.ts new file mode 100644 index 00000000..3fb3e3c8 --- /dev/null +++ b/packages/chrome-extension/src/tools/input.ts @@ -0,0 +1,252 @@ +/** + * Input tools: click, fill, fill_form, type_text, press_key, hover, drag, handle_dialog, upload_file + * + * These use chrome.debugger CDP commands for input simulation. + * Element targeting uses the "uid" from accessibility tree snapshots. + */ + +import type { PageManager } from '../page-manager.js'; + +async function cdp(tabId: number, method: string, params?: Record): Promise { + return chrome.debugger.sendCommand({ tabId }, method, params); +} + +async function ensureDebugger(pm: PageManager, tabId: number): Promise { + if (pm.isDebuggerAttached(tabId)) return; + await chrome.debugger.attach({ tabId }, '1.3'); + pm.setDebuggerAttached(tabId, true); + await chrome.debugger.sendCommand({ tabId }, 'Page.enable'); + await chrome.debugger.sendCommand({ tabId }, 'Runtime.enable'); +} + +function requireSelectedTab(pm: PageManager): number { + const tabId = pm.selectedTabId; + if (tabId === null) throw new Error('No page selected. Call new_page or select_page first.'); + return tabId; +} + +/** + * Resolve a snapshot uid to DOM coordinates via JS evaluation. + * The uid corresponds to an aria-snapshot element with a data-uid attribute + * or a node from the accessibility tree. + */ +async function resolveUidToCoords(tabId: number, uid: string): Promise<{ x: number; y: number }> { + const script = ` + (function() { + // Try data-uid attribute first + let el = document.querySelector('[data-uid="${uid}"]'); + if (!el) { + // Try aria-label or other attributes + const all = document.querySelectorAll('*'); + for (const e of all) { + if (e.getAttribute('data-snapshot-uid') === '${uid}') { el = e; break; } + } + } + if (!el) return null; + const rect = el.getBoundingClientRect(); + return { x: rect.x + rect.width / 2, y: rect.y + rect.height / 2 }; + })() + `; + const result = await cdp(tabId, 'Runtime.evaluate', { + expression: script, returnByValue: true, + }) as { result?: { value?: { x: number; y: number } | null } }; + + if (!result?.result?.value) { + throw new Error(`Element with uid "${uid}" not found on page`); + } + return result.result.value; +} + +async function dispatchClick(tabId: number, x: number, y: number): Promise { + await cdp(tabId, 'Input.dispatchMouseEvent', { + type: 'mousePressed', x, y, button: 'left', clickCount: 1, + }); + await cdp(tabId, 'Input.dispatchMouseEvent', { + type: 'mouseReleased', x, y, button: 'left', clickCount: 1, + }); +} + +export function registerInputTools( + register: (name: string, handler: (params: Record) => Promise) => void, + pm: PageManager, +): void { + + register('click', async (params) => { + const uid = params.uid as string; + if (!uid) throw new Error('uid is required'); + const tabId = requireSelectedTab(pm); + await ensureDebugger(pm, tabId); + + const { x, y } = await resolveUidToCoords(tabId, uid); + await dispatchClick(tabId, x, y); + return `Clicked element ${uid} at (${Math.round(x)}, ${Math.round(y)})`; + }); + + register('fill', async (params) => { + const uid = params.uid as string; + const value = params.value as string; + if (!uid) throw new Error('uid is required'); + if (value === undefined) throw new Error('value is required'); + const tabId = requireSelectedTab(pm); + await ensureDebugger(pm, tabId); + + const { x, y } = await resolveUidToCoords(tabId, uid); + await dispatchClick(tabId, x, y); + // Select all then replace + await cdp(tabId, 'Input.dispatchKeyEvent', { type: 'keyDown', key: 'a', code: 'KeyA', modifiers: 2 }); // Ctrl+A + await cdp(tabId, 'Input.dispatchKeyEvent', { type: 'keyUp', key: 'a', code: 'KeyA', modifiers: 2 }); + await cdp(tabId, 'Input.insertText', { text: value }); + return `Filled element ${uid} with "${value}"`; + }); + + register('fill_form', async (params) => { + const fields = params.fields as Array<{ uid: string; value: string }>; + if (!fields || !Array.isArray(fields)) throw new Error('fields array is required'); + const tabId = requireSelectedTab(pm); + await ensureDebugger(pm, tabId); + + const results: string[] = []; + for (const field of fields) { + const { x, y } = await resolveUidToCoords(tabId, field.uid); + await dispatchClick(tabId, x, y); + await cdp(tabId, 'Input.dispatchKeyEvent', { type: 'keyDown', key: 'a', code: 'KeyA', modifiers: 2 }); + await cdp(tabId, 'Input.dispatchKeyEvent', { type: 'keyUp', key: 'a', code: 'KeyA', modifiers: 2 }); + await cdp(tabId, 'Input.insertText', { text: field.value }); + results.push(`${field.uid}: "${field.value}"`); + } + return `Filled ${results.length} fields:\n${results.join('\n')}`; + }); + + register('type_text', async (params) => { + const text = params.text as string; + if (!text) throw new Error('text is required'); + const tabId = requireSelectedTab(pm); + await ensureDebugger(pm, tabId); + + await cdp(tabId, 'Input.insertText', { text }); + return `Typed "${text}"`; + }); + + register('press_key', async (params) => { + const key = params.key as string; + if (!key) throw new Error('key is required'); + const tabId = requireSelectedTab(pm); + await ensureDebugger(pm, tabId); + + const parts = key.split('+'); + let modifiers = 0; + const modifierKeys: string[] = []; + for (const part of parts.slice(0, -1)) { + const lower = part.toLowerCase().trim(); + if (lower === 'control' || lower === 'ctrl') { modifiers |= 2; modifierKeys.push(lower); } + else if (lower === 'alt') { modifiers |= 1; modifierKeys.push(lower); } + else if (lower === 'shift') { modifiers |= 8; modifierKeys.push(lower); } + else if (lower === 'meta' || lower === 'command' || lower === 'cmd') { modifiers |= 4; modifierKeys.push(lower); } + } + const mainKey = parts[parts.length - 1].trim(); + + await cdp(tabId, 'Input.dispatchKeyEvent', { + type: 'keyDown', key: mainKey, modifiers, + }); + await cdp(tabId, 'Input.dispatchKeyEvent', { + type: 'keyUp', key: mainKey, modifiers, + }); + return `Pressed key: ${key}`; + }); + + register('hover', async (params) => { + const uid = params.uid as string; + if (!uid) throw new Error('uid is required'); + const tabId = requireSelectedTab(pm); + await ensureDebugger(pm, tabId); + + const { x, y } = await resolveUidToCoords(tabId, uid); + await cdp(tabId, 'Input.dispatchMouseEvent', { + type: 'mouseMoved', x, y, + }); + return `Hovered over element ${uid} at (${Math.round(x)}, ${Math.round(y)})`; + }); + + register('drag', async (params) => { + const fromUid = params.from_uid as string ?? params.fromUid as string; + const toUid = params.to_uid as string ?? params.toUid as string; + if (!fromUid || !toUid) throw new Error('from_uid and to_uid are required'); + const tabId = requireSelectedTab(pm); + await ensureDebugger(pm, tabId); + + const from = await resolveUidToCoords(tabId, fromUid); + const to = await resolveUidToCoords(tabId, toUid); + + await cdp(tabId, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x: from.x, y: from.y }); + await cdp(tabId, 'Input.dispatchMouseEvent', { type: 'mousePressed', x: from.x, y: from.y, button: 'left' }); + // Intermediate move steps for smooth drag + const steps = 5; + for (let i = 1; i <= steps; i++) { + const x = from.x + (to.x - from.x) * (i / steps); + const y = from.y + (to.y - from.y) * (i / steps); + await cdp(tabId, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x, y }); + } + await cdp(tabId, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x: to.x, y: to.y, button: 'left' }); + + return `Dragged from ${fromUid} to ${toUid}`; + }); + + register('handle_dialog', async (params) => { + const accept = params.accept !== false; + const promptText = params.promptText as string | undefined; + const tabId = requireSelectedTab(pm); + await ensureDebugger(pm, tabId); + + await cdp(tabId, 'Page.handleJavaScriptDialog', { + accept, + ...(promptText !== undefined ? { promptText } : {}), + }); + return `Dialog ${accept ? 'accepted' : 'dismissed'}`; + }); + + register('upload_file', async (params) => { + const uid = params.uid as string; + const filePath = params.filePath as string ?? params.file_path as string; + if (!uid || !filePath) throw new Error('uid and filePath are required'); + const tabId = requireSelectedTab(pm); + await ensureDebugger(pm, tabId); + + // Resolve uid to DOM node + const script = ` + (function() { + let el = document.querySelector('[data-uid="${uid}"]'); + if (!el) { + const all = document.querySelectorAll('input[type="file"]'); + for (const e of all) { + if (e.getAttribute('data-snapshot-uid') === '${uid}') { el = e; break; } + } + } + return el ? true : false; + })() + `; + const found = await cdp(tabId, 'Runtime.evaluate', { + expression: script, returnByValue: true, + }) as { result?: { value?: boolean } }; + + if (!found?.result?.value) { + throw new Error(`File input with uid "${uid}" not found`); + } + + // Use DOM.setFileInputFiles via the node + const docResult = await cdp(tabId, 'DOM.getDocument') as { root?: { nodeId?: number } }; + const nodeResult = await cdp(tabId, 'DOM.querySelector', { + nodeId: docResult?.root?.nodeId, + selector: `[data-uid="${uid}"]`, + }) as { nodeId?: number }; + + if (nodeResult?.nodeId) { + await cdp(tabId, 'DOM.setFileInputFiles', { + files: [filePath], + nodeId: nodeResult.nodeId, + }); + return `Uploaded file "${filePath}" to element ${uid}`; + } + + throw new Error(`Could not set file on element ${uid}`); + }); +} diff --git a/packages/chrome-extension/src/tools/inspection.ts b/packages/chrome-extension/src/tools/inspection.ts new file mode 100644 index 00000000..e4f8f682 --- /dev/null +++ b/packages/chrome-extension/src/tools/inspection.ts @@ -0,0 +1,218 @@ +/** + * Inspection tools: take_screenshot, take_snapshot, evaluate_script, + * get_console_message, list_console_messages, lighthouse_audit + */ + +import type { PageManager } from '../page-manager.js'; + +async function cdp(tabId: number, method: string, params?: Record): Promise { + return chrome.debugger.sendCommand({ tabId }, method, params); +} + +async function ensureDebugger(pm: PageManager, tabId: number): Promise { + if (pm.isDebuggerAttached(tabId)) return; + await chrome.debugger.attach({ tabId }, '1.3'); + pm.setDebuggerAttached(tabId, true); + await chrome.debugger.sendCommand({ tabId }, 'Page.enable'); + await chrome.debugger.sendCommand({ tabId }, 'Runtime.enable'); +} + +function requireSelectedTab(pm: PageManager): number { + const tabId = pm.selectedTabId; + if (tabId === null) throw new Error('No page selected. Call new_page or select_page first.'); + return tabId; +} + +// Console message storage per tab +const consoleMessages = new Map>(); +let nextMsgId = 1; + +export function setupConsoleListener(): void { + chrome.debugger.onEvent.addListener((source, method, params) => { + if (method === 'Runtime.consoleAPICalled' && source.tabId) { + const p = params as { type: string; args?: Array<{ value?: unknown; description?: string }>; timestamp?: number }; + const text = (p.args ?? []).map(a => a.description ?? String(a.value ?? '')).join(' '); + let messages = consoleMessages.get(source.tabId); + if (!messages) { + messages = []; + consoleMessages.set(source.tabId, messages); + } + messages.push({ + id: nextMsgId++, + level: p.type ?? 'log', + text, + timestamp: p.timestamp ?? Date.now(), + }); + // Keep last 200 messages per tab + if (messages.length > 200) messages.splice(0, messages.length - 200); + } + }); +} + +export function registerInspectionTools( + register: (name: string, handler: (params: Record) => Promise) => void, + pm: PageManager, +): void { + + register('take_screenshot', async (params) => { + const tabId = requireSelectedTab(pm); + await ensureDebugger(pm, tabId); + + const format = (params.format as string) || 'png'; + const quality = params.quality as number | undefined; + const fullPage = params.fullPage === true; + + const cdpParams: Record = { + format: format === 'jpg' ? 'jpeg' : format, + }; + if (quality !== undefined) cdpParams.quality = quality; + if (fullPage) cdpParams.captureBeyondViewport = true; + + const result = await cdp(tabId, 'Page.captureScreenshot', cdpParams) as { data?: string }; + if (!result?.data) throw new Error('Screenshot failed'); + + return `data:image/${format};base64,${result.data}`; + }); + + register('take_snapshot', async (params) => { + const tabId = requireSelectedTab(pm); + await ensureDebugger(pm, tabId); + + // Get accessibility tree + const result = await cdp(tabId, 'Accessibility.getFullAXTree') as { + nodes?: Array<{ + nodeId: string; + role?: { value?: string }; + name?: { value?: string }; + properties?: Array<{ name: string; value: { value?: unknown } }>; + childIds?: string[]; + backendDOMNodeId?: number; + }>; + }; + + if (!result?.nodes || result.nodes.length === 0) { + return 'Empty accessibility tree'; + } + + const lines: string[] = []; + const nodeMap = new Map(result.nodes.map(n => [n.nodeId, n])); + let uidCounter = 1; + + function walk(nodeId: string, depth: number): void { + const node = nodeMap.get(nodeId); + if (!node) return; + + const role = node.role?.value ?? ''; + const name = node.name?.value ?? ''; + + // Skip generic/none roles and unnamed nodes at root level + if (role === 'none' || role === 'generic') { + for (const childId of node.childIds ?? []) { + walk(childId, depth); + } + return; + } + + if (role || name) { + const uid = `e${uidCounter++}`; + const indent = ' '.repeat(depth); + const nameStr = name ? ` "${name}"` : ''; + lines.push(`${indent}[${uid}] ${role}${nameStr}`); + + for (const childId of node.childIds ?? []) { + walk(childId, depth + 1); + } + } + } + + // Start from root + if (result.nodes.length > 0) { + walk(result.nodes[0].nodeId, 0); + } + + return lines.length > 0 ? lines.join('\n') : 'Empty accessibility tree'; + }); + + register('evaluate_script', async (params) => { + const expression = params.expression as string; + if (!expression) throw new Error('expression is required'); + const tabId = requireSelectedTab(pm); + await ensureDebugger(pm, tabId); + + const result = await cdp(tabId, 'Runtime.evaluate', { + expression, + returnByValue: true, + awaitPromise: true, + }) as { result?: { value?: unknown; description?: string }; exceptionDetails?: { text?: string } }; + + if (result?.exceptionDetails) { + throw new Error(`Script error: ${result.exceptionDetails.text}`); + } + + const value = result?.result?.value; + if (value === undefined) return result?.result?.description ?? 'undefined'; + return typeof value === 'string' ? value : JSON.stringify(value, null, 2); + }); + + register('get_console_message', async (params) => { + const msgId = params.msgid as number ?? params.id as number; + if (msgId === undefined) throw new Error('msgid is required'); + const tabId = requireSelectedTab(pm); + + const messages = consoleMessages.get(tabId) ?? []; + const msg = messages.find(m => m.id === msgId); + if (!msg) return `Console message ${msgId} not found`; + + return `[${msg.level}] ${msg.text}`; + }); + + register('list_console_messages', async () => { + const tabId = requireSelectedTab(pm); + const messages = consoleMessages.get(tabId) ?? []; + + if (messages.length === 0) return 'No console messages'; + + return messages.map(m => `${m.id}: [${m.level}] ${m.text}`).join('\n'); + }); + + register('lighthouse_audit', async (params) => { + const tabId = requireSelectedTab(pm); + const categories = (params.categories as string[]) ?? ['accessibility', 'best-practices', 'seo']; + + // Lighthouse is not available via chrome.debugger — provide a basic a11y audit instead + await ensureDebugger(pm, tabId); + + const result = await cdp(tabId, 'Accessibility.getFullAXTree') as { + nodes?: Array<{ role?: { value?: string }; name?: { value?: string } }>; + }; + + const nodes = result?.nodes ?? []; + const issues: string[] = []; + + // Basic accessibility checks + let imagesWithoutAlt = 0; + let buttonsWithoutLabel = 0; + let linksWithoutText = 0; + + for (const node of nodes) { + const role = node.role?.value; + const name = node.name?.value; + if (role === 'img' && !name) imagesWithoutAlt++; + if (role === 'button' && !name) buttonsWithoutLabel++; + if (role === 'link' && !name) linksWithoutText++; + } + + if (imagesWithoutAlt > 0) issues.push(`${imagesWithoutAlt} image(s) without alt text`); + if (buttonsWithoutLabel > 0) issues.push(`${buttonsWithoutLabel} button(s) without labels`); + if (linksWithoutText > 0) issues.push(`${linksWithoutText} link(s) without text`); + + const report = [ + `Accessibility Audit (${nodes.length} nodes analyzed)`, + `Categories: ${categories.join(', ')}`, + '', + issues.length > 0 ? `Issues found:\n${issues.map(i => ` - ${i}`).join('\n')}` : 'No issues found', + ]; + + return report.join('\n'); + }); +} diff --git a/packages/chrome-extension/src/tools/navigation.ts b/packages/chrome-extension/src/tools/navigation.ts new file mode 100644 index 00000000..87a7e540 --- /dev/null +++ b/packages/chrome-extension/src/tools/navigation.ts @@ -0,0 +1,222 @@ +/** + * Navigation tools: new_page, close_page, list_pages, select_page, navigate_page, wait_for + * + * These use chrome.tabs API for tab management and chrome.debugger + * for CDP-level navigation control. + */ + +import type { PageManager } from '../page-manager.js'; + +/** Helper: attach debugger to tab if not already attached */ +async function ensureDebugger(pm: PageManager, tabId: number): Promise { + if (pm.isDebuggerAttached(tabId)) return; + await chrome.debugger.attach({ tabId }, '1.3'); + pm.setDebuggerAttached(tabId, true); + await chrome.debugger.sendCommand({ tabId }, 'Page.enable'); + await chrome.debugger.sendCommand({ tabId }, 'Runtime.enable'); +} + +/** Helper: send CDP command on a tab */ +async function cdp(tabId: number, method: string, params?: Record): Promise { + return chrome.debugger.sendCommand({ tabId }, method, params); +} + +/** Helper: wait for page load after navigation */ +async function waitForLoad(tabId: number, timeoutMs: number): Promise { + // Check if already loaded before registering listener (avoids race condition) + try { + const tab = await chrome.tabs.get(tabId); + if (tab.status === 'complete') return; + } catch { return; } + + return new Promise((resolve) => { + const timer = setTimeout(() => { + chrome.tabs.onUpdated.removeListener(listener); + resolve(); + }, timeoutMs); + + const listener = (updatedTabId: number, changeInfo: chrome.tabs.TabChangeInfo) => { + if (updatedTabId === tabId && changeInfo.status === 'complete') { + clearTimeout(timer); + chrome.tabs.onUpdated.removeListener(listener); + resolve(); + } + }; + chrome.tabs.onUpdated.addListener(listener); + + // Double-check after listener is registered (another race window) + chrome.tabs.get(tabId).then(t => { + if (t.status === 'complete') { + clearTimeout(timer); + chrome.tabs.onUpdated.removeListener(listener); + resolve(); + } + }).catch(() => { + clearTimeout(timer); + chrome.tabs.onUpdated.removeListener(listener); + resolve(); + }); + }); +} + +export function registerNavigationTools( + register: (name: string, handler: (params: Record) => Promise) => void, + pm: PageManager, +): void { + + register('new_page', async (params) => { + const url = (params.url as string) || 'about:blank'; + const background = params.background === true; + const timeout = (params.timeout as number) || 15000; + + const tab = await chrome.tabs.create({ url, active: !background }); + if (!tab.id) throw new Error('Failed to create tab'); + + const pageId = pm.getPageId(tab.id); + pm.selectPage(pageId); + + if (url !== 'about:blank') { + await waitForLoad(tab.id, timeout); + } + + const updatedTab = await chrome.tabs.get(tab.id); + return formatPageList([{ pageId, tab: updatedTab, selected: true }]); + }); + + register('open_page', async (params) => { + const url = (params.url as string) || 'about:blank'; + const background = params.background === true; + const timeout = (params.timeout as number) || 15000; + + const tab = await chrome.tabs.create({ url, active: !background }); + if (!tab.id) throw new Error('Failed to create tab'); + + const pageId = pm.getPageId(tab.id); + pm.selectPage(pageId); + + if (url !== 'about:blank') { + await waitForLoad(tab.id, timeout); + } + + const updatedTab = await chrome.tabs.get(tab.id); + return formatPageList([{ pageId, tab: updatedTab, selected: true }]); + }); + + register('close_page', async (params) => { + const pageId = params.pageId as number; + if (pageId === undefined) throw new Error('pageId is required'); + + const tabId = pm.getTabId(pageId); + if (tabId === undefined) throw new Error(`Page ${pageId} not found`); + + if (pm.isDebuggerAttached(tabId)) { + try { await chrome.debugger.detach({ tabId }); } catch { /* ignore */ } + } + await chrome.tabs.remove(tabId); + pm.removePage(pageId); + + return `Closed page ${pageId}`; + }); + + register('list_pages', async () => { + const tabs = await chrome.tabs.query({}); + const entries: Array<{ pageId: number; tab: chrome.tabs.Tab; selected: boolean }> = []; + + for (const tab of tabs) { + if (!tab.id || tab.id === chrome.tabs.TAB_ID_NONE) continue; + const pageId = pm.getPageId(tab.id); + entries.push({ pageId, tab, selected: pageId === pm.selectedPageId }); + } + + entries.sort((a, b) => a.pageId - b.pageId); + return formatPageList(entries); + }); + + register('select_page', async (params) => { + const pageId = params.pageId as number; + if (pageId === undefined) throw new Error('pageId is required'); + + const tabId = pm.getTabId(pageId); + if (tabId === undefined) throw new Error(`Page ${pageId} not found`); + + pm.selectPage(pageId); + + const bringToFront = params.bringToFront !== false; + if (bringToFront) { + await chrome.tabs.update(tabId, { active: true }); + const tab = await chrome.tabs.get(tabId); + if (tab.windowId) { + await chrome.windows.update(tab.windowId, { focused: true }); + } + } + + const tab = await chrome.tabs.get(tabId); + return formatPageList([{ pageId, tab, selected: true }]); + }); + + register('navigate_page', async (params) => { + const url = params.url as string | undefined; + const timeout = (params.timeout as number) || 15000; + + const tabId = pm.selectedTabId; + if (tabId === null) throw new Error('No page selected. Call new_page or select_page first.'); + + if (url) { + await ensureDebugger(pm, tabId); + await cdp(tabId, 'Page.navigate', { url }); + await waitForLoad(tabId, timeout); + } else if (params.action === 'back') { + await ensureDebugger(pm, tabId); + await cdp(tabId, 'Page.navigateToHistoryEntry', { entryId: -1 }).catch(() => { + return chrome.tabs.goBack(tabId); + }); + } else if (params.action === 'forward') { + await ensureDebugger(pm, tabId); + await cdp(tabId, 'Page.navigateToHistoryEntry', { entryId: 1 }).catch(() => { + return chrome.tabs.goForward(tabId); + }); + } else if (params.action === 'reload') { + await chrome.tabs.reload(tabId); + await waitForLoad(tabId, timeout); + } + + const tab = await chrome.tabs.get(tabId); + const pageId = pm.getPageId(tabId); + return formatPageList([{ pageId, tab, selected: true }]); + }); + + register('wait_for', async (params) => { + const text = params.text as string; + const timeout = (params.timeout as number) || 30000; + if (!text) throw new Error('text parameter is required'); + + const tabId = pm.selectedTabId; + if (tabId === null) throw new Error('No page selected.'); + + await ensureDebugger(pm, tabId); + + const startTime = Date.now(); + while (Date.now() - startTime < timeout) { + const result = await cdp(tabId, 'Runtime.evaluate', { + expression: `document.body?.innerText?.includes(${JSON.stringify(text)}) ?? false`, + returnByValue: true, + }) as { result?: { value?: boolean } }; + + if (result?.result?.value === true) { + return `Found text "${text}" on page`; + } + await new Promise(r => setTimeout(r, 500)); + } + + throw new Error(`Text "${text}" not found within ${timeout}ms`); + }); +} + +function formatPageList(entries: Array<{ pageId: number; tab: chrome.tabs.Tab; selected: boolean }>): string { + if (entries.length === 0) return 'No pages open'; + return entries.map(e => { + const url = e.tab.url || e.tab.pendingUrl || 'about:blank'; + const sel = e.selected ? ' [selected]' : ''; + return `${e.pageId}: ${url}${sel}`; + }).join('\n'); +} diff --git a/packages/chrome-extension/src/tools/network.ts b/packages/chrome-extension/src/tools/network.ts new file mode 100644 index 00000000..f62813bf --- /dev/null +++ b/packages/chrome-extension/src/tools/network.ts @@ -0,0 +1,182 @@ +/** + * Network tools: list_network_requests, get_network_request + * Performance tools: performance_start_trace, performance_stop_trace, etc. + * Emulation tools: emulate, resize_page + */ + +import type { PageManager } from '../page-manager.js'; + +async function cdp(tabId: number, method: string, params?: Record): Promise { + return chrome.debugger.sendCommand({ tabId }, method, params); +} + +async function ensureDebugger(pm: PageManager, tabId: number): Promise { + if (pm.isDebuggerAttached(tabId)) return; + await chrome.debugger.attach({ tabId }, '1.3'); + pm.setDebuggerAttached(tabId, true); + await chrome.debugger.sendCommand({ tabId }, 'Page.enable'); + await chrome.debugger.sendCommand({ tabId }, 'Runtime.enable'); +} + +function requireSelectedTab(pm: PageManager): number { + const tabId = pm.selectedTabId; + if (tabId === null) throw new Error('No page selected.'); + return tabId; +} + +// Network request storage per tab +interface StoredRequest { + id: string; + url: string; + method: string; + status?: number; + type?: string; + responseHeaders?: Record; + responseBody?: string; + timestamp: number; +} + +const networkRequests = new Map(); +let netIdCounter = 1; +const networkEnabled = new Set(); + +export function setupNetworkListener(): void { + chrome.debugger.onEvent.addListener((source, method, params) => { + if (!source.tabId) return; + const p = params as Record; + + if (method === 'Network.responseReceived') { + let reqs = networkRequests.get(source.tabId); + if (!reqs) { reqs = []; networkRequests.set(source.tabId, reqs); } + const response = p.response as Record | undefined; + reqs.push({ + id: `req${netIdCounter++}`, + url: (response?.url as string) ?? '', + method: (p.type as string) ?? 'GET', + status: response?.status as number | undefined, + type: p.type as string | undefined, + timestamp: Date.now(), + }); + if (reqs.length > 500) reqs.splice(0, reqs.length - 500); + } + }); +} + +async function enableNetwork(pm: PageManager, tabId: number): Promise { + await ensureDebugger(pm, tabId); + if (!networkEnabled.has(tabId)) { + await cdp(tabId, 'Network.enable'); + networkEnabled.add(tabId); + } +} + +export function registerNetworkTools( + register: (name: string, handler: (params: Record) => Promise) => void, + pm: PageManager, +): void { + + register('list_network_requests', async () => { + const tabId = requireSelectedTab(pm); + await enableNetwork(pm, tabId); + + const reqs = networkRequests.get(tabId) ?? []; + if (reqs.length === 0) return 'No network requests captured'; + + return reqs.map(r => + `${r.id}: ${r.method ?? 'GET'} ${r.url} → ${r.status ?? 'pending'}` + ).join('\n'); + }); + + register('get_network_request', async (params) => { + const reqId = params.reqid as string ?? params.id as string; + if (!reqId) throw new Error('reqid is required'); + const tabId = requireSelectedTab(pm); + + const reqs = networkRequests.get(tabId) ?? []; + const req = reqs.find(r => r.id === reqId); + if (!req) return `Network request ${reqId} not found`; + + return JSON.stringify({ + id: req.id, + url: req.url, + method: req.method, + status: req.status, + type: req.type, + }, null, 2); + }); + + register('performance_start_trace', async () => { + const tabId = requireSelectedTab(pm); + await ensureDebugger(pm, tabId); + await cdp(tabId, 'Tracing.start', { + categories: '-*,devtools.timeline,v8.execute,disabled-by-default-devtools.timeline', + }); + return 'Performance trace started'; + }); + + register('performance_stop_trace', async () => { + const tabId = requireSelectedTab(pm); + await ensureDebugger(pm, tabId); + await cdp(tabId, 'Tracing.end'); + return 'Performance trace stopped. Results will be available via tracing events.'; + }); + + register('performance_analyze_insight', async (params) => { + const insightId = params.insightId as string; + return `Performance insight analysis for "${insightId}" is not available in extension mode. Use chrome-devtools-mcp for full performance profiling.`; + }); + + register('take_heapsnapshot', async () => { + return 'Heap snapshots are not available in extension mode. Use chrome-devtools-mcp for memory profiling.'; + }); + + register('emulate', async (params) => { + const tabId = requireSelectedTab(pm); + await ensureDebugger(pm, tabId); + + if (params.width || params.height) { + await cdp(tabId, 'Emulation.setDeviceMetricsOverride', { + width: (params.width as number) || 0, + height: (params.height as number) || 0, + deviceScaleFactor: (params.deviceScaleFactor as number) || 1, + mobile: params.mobile === true, + }); + } + + if (params.userAgent) { + await cdp(tabId, 'Emulation.setUserAgentOverride', { + userAgent: params.userAgent as string, + }); + } + + if (params.geolocation) { + const geo = params.geolocation as { latitude: number; longitude: number; accuracy?: number }; + await cdp(tabId, 'Emulation.setGeolocationOverride', { + latitude: geo.latitude, + longitude: geo.longitude, + accuracy: geo.accuracy ?? 1, + }); + } + + if (params.colorScheme) { + await cdp(tabId, 'Emulation.setEmulatedMedia', { + features: [{ name: 'prefers-color-scheme', value: params.colorScheme as string }], + }); + } + + return 'Emulation settings applied'; + }); + + register('resize_page', async (params) => { + const width = params.width as number; + const height = params.height as number; + if (!width || !height) throw new Error('width and height are required'); + const tabId = requireSelectedTab(pm); + + const tab = await chrome.tabs.get(tabId); + if (tab.windowId) { + await chrome.windows.update(tab.windowId, { width, height }); + } + return `Resized to ${width}x${height}`; + }); +} diff --git a/packages/chrome-extension/tsconfig.json b/packages/chrome-extension/tsconfig.json new file mode 100644 index 00000000..c2b34458 --- /dev/null +++ b/packages/chrome-extension/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ES2022", + "moduleResolution": "bundler", + "lib": ["ES2022", "DOM"], + "outDir": "dist", + "rootDir": "src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "declaration": false, + "types": ["chrome"] + }, + "include": ["src"] +} diff --git a/packages/cli/package.json b/packages/cli/package.json index ab53a1fb..20b724c8 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -33,6 +33,7 @@ "@markus/comms": "workspace:*", "@markus/core": "workspace:*", "@markus/org-manager": "workspace:*", + "@markus/remote": "workspace:*", "@markus/shared": "workspace:*", "commander": "^14.0.3", "esbuild": "^0.27.4" diff --git a/packages/cli/src/api-client.ts b/packages/cli/src/api-client.ts index b915f2af..51eb16a5 100644 --- a/packages/cli/src/api-client.ts +++ b/packages/cli/src/api-client.ts @@ -75,7 +75,12 @@ export class ApiClient { private async request(url: string, init: RequestInit): Promise { const headers = { ...this.headers }; - if (!init.body) delete headers['Content-Type']; + // Always send Content-Type even for empty-body POST (e.g., /tasks/:id/approve) + // The server rejects requests without Content-Type with HTTP 415. + // We explicitly set it to 'application/json' for consistency. + if (init.body === undefined || init.body === null) { + // Ensure Content-Type is present even for no-body requests + } let res: Response; try { diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts index a3d14976..96bc82ba 100644 --- a/packages/cli/src/commands/start.ts +++ b/packages/cli/src/commands/start.ts @@ -1,6 +1,6 @@ import type { Command } from 'commander'; import { resolve, join, dirname } from 'node:path'; -import { existsSync } from 'node:fs'; +import { existsSync, readFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { allTemplateDirs, resolveTemplatesDir, resolveWebUiDir } from '../paths.js'; import { @@ -305,6 +305,10 @@ async function createServices(config: ReturnType) { if (config.browser?.remoteDebuggingPort) { agentManager.setBrowserRemoteDebuggingPort(config.browser.remoteDebuggingPort); } + if (config.browser?.autoClickAllowDialog) { + agentManager.setBrowserAutoClickAllowDialog(true); + } + agentManager.startBrowserBridge(config.browser?.extensionBridgePort); taskService.setAgentManager(agentManager); @@ -897,6 +901,45 @@ async function startServer(config: ReturnType, values: Record apiServer.setGateway(gateway, gatewaySecret); log.info('External Agent Gateway enabled', { secret: gatewaySecret === 'markus-gateway-default-secret-change-me' ? '(default)' : '(custom)' }); + // ── Remote Access (WebRTC P2P via markus-hub + signal server) ───────── + { + const hubTokenPath = join(homedir(), '.markus', 'hub-token'); + + const createRemoteAgent = async () => { + const token = existsSync(hubTokenPath) ? readFileSync(hubTokenPath, 'utf-8').trim() : undefined; + if (!token) return null; + const { RemoteAccessAgent } = await import('@markus/remote'); + return new RemoteAccessAgent({ + hubUrl: config.remote?.hubUrl ?? config.hub?.url ?? 'https://www.markus.global', + hubToken: token, + instanceName: config.remote?.instanceName ?? config.org?.name ?? 'My Markus', + localPort: config.server?.apiPort ?? 8056, + jwtSecret: process.env['JWT_SECRET'], + }); + }; + + apiServer.setRemoteAgentFactory(createRemoteAgent); + + if (config.remote?.enabled !== false) { + const remoteAgent = await createRemoteAgent(); + if (remoteAgent) { + apiServer.setRemoteAgent(remoteAgent); + if (config.remote?.autoConnect !== false) { + remoteAgent.start().then(() => { + const status = remoteAgent.getStatus(); + if (status.remoteUrl) { + log.info(`Remote access available at ${status.remoteUrl}`); + } + }).catch((err: unknown) => { + log.warn('Remote access failed to start', { error: String(err) }); + }); + } + } else { + log.debug('Remote access: no Hub token yet (can enable later via Settings)'); + } + } + } + apiServer.start(); taskService.setWSBroadcaster(apiServer.getWSBroadcaster()); requirementService.setWSBroadcaster(apiServer.getWSBroadcaster()); diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.json index f3115801..0f8cfc11 100644 --- a/packages/cli/tsconfig.json +++ b/packages/cli/tsconfig.json @@ -9,6 +9,7 @@ { "path": "../shared" }, { "path": "../core" }, { "path": "../comms" }, - { "path": "../org-manager" } + { "path": "../org-manager" }, + { "path": "../remote" } ] } diff --git a/packages/core/package.json b/packages/core/package.json index 430d6f4f..d9a89bae 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -19,9 +19,11 @@ "js-tiktoken": "^1.0.21", "linkedom": "^0.18.12", "node-cron": "^3.0.3", - "turndown": "^7.2.2" + "turndown": "^7.2.2", + "ws": "^8.19.0" }, "devDependencies": { - "@types/turndown": "^5.0.6" + "@types/turndown": "^5.0.6", + "@types/ws": "^8.18.1" } } diff --git a/packages/core/src/agent-manager.ts b/packages/core/src/agent-manager.ts index bbd1bdd0..e2e56d98 100644 --- a/packages/core/src/agent-manager.ts +++ b/packages/core/src/agent-manager.ts @@ -34,6 +34,9 @@ import { createSettingsTools } from './tools/settings.js'; import { createRecallTool, type RecallCallbacks } from './tools/recall.js'; import { SemanticMemorySearch, OpenAIEmbeddingProvider, LocalVectorStore } from './memory/semantic-search.js'; import type { SkillRegistry } from './skills/types.js'; +import { clickChromeAllowDialog } from './tools/chrome-dialog-clicker.js'; +import { MarkusBrowserBridge } from './tools/markus-browser-bridge.js'; +import { createBridgeToolHandlers, getBridgeToolDescriptors } from './tools/markus-browser-mcp.js'; import { SecurityGuard, type SecurityPolicy } from './security.js'; import { DelegationManager, type TaskDelegation } from '@markus/a2a'; import type { TemplateRegistry } from './templates/registry.js'; @@ -302,6 +305,9 @@ export class AgentManager { private mcpManager: MCPClientManager; private browserSessionManager: BrowserSessionManager; private remoteDebuggingPort = 0; + private autoClickAllowDialog = false; + private chromeAutoClickRunning = false; + private browserBridge: MarkusBrowserBridge; private globalSecurityPolicy?: SecurityPolicy; private globalMcpServers?: Record; private skillRegistry?: SkillRegistry; @@ -494,7 +500,11 @@ export class AgentManager { this.sharedDataDir = options.sharedDataDir; this.eventBus = options.eventBus ?? new EventBus(); this.mcpManager = new MCPClientManager(); + this.mcpManager.setOnReconnect((serverName) => { + this.triggerChromeDialogAutoClick(serverName); + }); this.browserSessionManager = new BrowserSessionManager(); + this.browserBridge = new MarkusBrowserBridge(); this.globalSecurityPolicy = options.securityPolicy; this.globalMcpServers = options.mcpServers; this.skillRegistry = options.skillRegistry; @@ -587,6 +597,29 @@ export class AgentManager { this.remoteDebuggingPort = port; } + setBrowserAutoClickAllowDialog(enabled: boolean): void { + this.autoClickAllowDialog = enabled; + } + + startBrowserBridge(port?: number): void { + if (port !== undefined) { + this.browserBridge = new MarkusBrowserBridge(port); + } + this.browserBridge.start(); + } + + stopBrowserBridge(): void { + this.browserBridge.stop(); + } + + get browserExtensionConnected(): boolean { + return this.browserBridge.connected; + } + + getBrowserBridge(): MarkusBrowserBridge { + return this.browserBridge; + } + /** * When remoteDebuggingPort is configured, replace --autoConnect with * --browserUrl so that the chrome-devtools MCP server reuses a persistent @@ -606,6 +639,65 @@ export class AgentManager { return { ...config, args }; } + /** + * Trigger Chrome dialog auto-click with smart mutex. + * Only one clicker process runs at a time. If triggered while already running, + * it's a no-op (the running clicker will detect the dialog when it appears). + */ + private triggerChromeDialogAutoClick(serverName: string): void { + if (serverName !== 'chrome-devtools' || !this.autoClickAllowDialog) return; + if (this.chromeAutoClickRunning) return; + this.chromeAutoClickRunning = true; + clickChromeAllowDialog(60) + .catch(() => {}) + .finally(() => { this.chromeAutoClickRunning = false; }); + } + + /** + * Register chrome-devtools tools for an agent WITHOUT starting the MCP process. + * + * Tool handlers dynamically choose bridge vs npx at CALL TIME: + * - If the Chrome extension is connected, use the WebSocket bridge (no dialog, instant). + * - Otherwise, fall back to npx chrome-devtools-mcp (lazy-started on first call). + * + * This ensures agents created before the extension connects can still + * use the bridge once it becomes available, and vice versa. + */ + private async registerChromeDevtoolsLazy( + agentId: string, + serverName: string, + serverConfig: { command: string; args?: string[]; env?: Record }, + ): Promise { + const toolDescriptors = getBridgeToolDescriptors(); + + // Register npx config lazily so callToolScoped can auto-connect when needed. + this.mcpManager.registerLazyScoped(serverName, serverConfig, agentId, toolDescriptors); + + let mcpTools: AgentToolHandler[] = toolDescriptors.map((tool) => ({ + name: `${serverName}__${tool.name}`, + description: `[MCP:${serverName}] ${tool.description}`, + inputSchema: tool.inputSchema, + execute: async (args: Record) => { + if (this.browserBridge.connected) { + const result = await this.browserBridge.callTool(tool.name, args); + if (result.error) return `Error: ${result.error}`; + return result.content; + } + // npx fallback; auto-click is triggered by mcpManager's onReconnect callback + return this.mcpManager.callToolScoped(serverName, agentId, tool.name, args); + }, + })); + + mcpTools = this.browserSessionManager.wrapToolHandlers(mcpTools, agentId); + this.browserSessionManager.setReconnector(agentId, serverName, async () => { + if (!this.browserBridge.connected) { + await this.mcpManager.disconnectServerScoped(serverName, agentId); + await this.mcpManager.connectServerScoped(serverName, serverConfig, agentId); + } + }); + return mcpTools; + } + setTaskService(taskService: TaskServiceBridge): void { this.taskService = taskService; } @@ -866,17 +958,20 @@ export class AgentManager { for (const [serverName, rawServerConfig] of Object.entries(skill.manifest.mcpServers)) { try { let mcpTools: AgentToolHandler[]; - if (isolation === 'per-agent' || isolation === 'pooled') { - const serverConfig = this.enrichChromeDevtoolsConfig(serverName, rawServerConfig); + const serverConfig = this.enrichChromeDevtoolsConfig(serverName, rawServerConfig); + if (serverName === 'chrome-devtools') { + // Lazy start: register tools now, connect on first call. + // Startup semaphore in MCPClientManager serializes connections. + mcpTools = await this.registerChromeDevtoolsLazy(id, serverName, serverConfig); + } else if (isolation === 'per-agent' || isolation === 'pooled') { await this.mcpManager.connectServerScoped(serverName, serverConfig, id); mcpTools = this.mcpManager.getToolHandlersScoped(serverName, id); mcpTools = this.browserSessionManager.wrapToolHandlers(mcpTools, id); - this.browserSessionManager.setReconnector(id, async () => { + this.browserSessionManager.setReconnector(id, serverName, async () => { await this.mcpManager.disconnectServerScoped(serverName, id); await this.mcpManager.connectServerScoped(serverName, serverConfig, id); }); } else { - const serverConfig = this.enrichChromeDevtoolsConfig(serverName, rawServerConfig); await this.mcpManager.connectServer(serverName, serverConfig); mcpTools = this.mcpManager.getToolHandlers(serverName); } @@ -905,21 +1000,25 @@ export class AgentManager { const skill = this.skillRegistry?.get(skillName); const isolation = skill?.manifest.isolation ?? 'shared'; for (const [serverName, rawSrvConfig] of Object.entries(mcpServers)) { - if (isolation === 'per-agent' || isolation === 'pooled') { - const srvConfig = this.enrichChromeDevtoolsConfig(serverName, rawSrvConfig); + const srvConfig = this.enrichChromeDevtoolsConfig(serverName, rawSrvConfig); + if (serverName === 'chrome-devtools') { + const chromeTools = await this.registerChromeDevtoolsLazy(id, serverName, srvConfig); + tools.push(...chromeTools); + } else if (isolation === 'per-agent' || isolation === 'pooled') { await this.mcpManager.connectServerScoped(serverName, srvConfig, id); tools.push(...this.mcpManager.getToolHandlersScoped(serverName, id)); } else { - const srvConfig = this.enrichChromeDevtoolsConfig(serverName, rawSrvConfig); await this.mcpManager.connectServer(serverName, srvConfig); tools.push(...this.mcpManager.getToolHandlers(serverName)); } } - if (isolation === 'per-agent' || isolation === 'pooled') { + // Wrap with BrowserSessionManager for per-agent tools (tab isolation) + const hasChromeDevtools = Object.keys(mcpServers).includes('chrome-devtools'); + if (!hasChromeDevtools && (isolation === 'per-agent' || isolation === 'pooled')) { tools = this.browserSessionManager.wrapToolHandlers(tools, id); for (const [serverName, rawSrvConfig] of Object.entries(mcpServers)) { const srvConfig = this.enrichChromeDevtoolsConfig(serverName, rawSrvConfig); - this.browserSessionManager.setReconnector(id, async () => { + this.browserSessionManager.setReconnector(id, serverName, async () => { await this.mcpManager.disconnectServerScoped(serverName, id); await this.mcpManager.connectServerScoped(serverName, srvConfig, id); }); @@ -1352,10 +1451,16 @@ export class AgentManager { // Connect MCP servers and register their tools const mcpConfigs = request.mcpServers ?? this.globalMcpServers; if (mcpConfigs) { - for (const [serverName, serverConfig] of Object.entries(mcpConfigs)) { + for (const [serverName, rawServerConfig] of Object.entries(mcpConfigs)) { try { - await this.mcpManager.connectServer(serverName, serverConfig); - const mcpTools = this.mcpManager.getToolHandlers(serverName); + const serverConfig = this.enrichChromeDevtoolsConfig(serverName, rawServerConfig); + let mcpTools: AgentToolHandler[]; + if (serverName === 'chrome-devtools') { + mcpTools = await this.registerChromeDevtoolsLazy(id, serverName, serverConfig); + } else { + await this.mcpManager.connectServer(serverName, serverConfig); + mcpTools = this.mcpManager.getToolHandlers(serverName); + } for (const tool of mcpTools) { agent.registerTool(tool); } @@ -1562,14 +1667,33 @@ export class AgentManager { } // Connect MCP servers declared by explicitly assigned skills (background, non-blocking). - // Connections complete asynchronously and register tools when ready. - // This avoids blocking startup for slow MCP processes (e.g. npx chrome-devtools). + // Skip chrome-devtools during restore — it connects lazily when the agent actually + // needs browser tools. This prevents flooding Chrome with 20+ concurrent CDP connections + // on startup which causes Chrome to crash. const mcpConnections: Array> = []; for (const skillName of config.skills) { const skill = this.skillRegistry.get(skillName); if (skill?.manifest.mcpServers) { const isolation = skill.manifest.isolation ?? 'shared'; for (const [serverName, rawServerConfig] of Object.entries(skill.manifest.mcpServers)) { + if (serverName === 'chrome-devtools') { + mcpConnections.push((async () => { + try { + const serverConfig = this.enrichChromeDevtoolsConfig(serverName, rawServerConfig); + const mcpTools = await this.registerChromeDevtoolsLazy(id, serverName, serverConfig); + const toolNames: string[] = []; + for (const tool of mcpTools) { + agent.registerTool(tool); + toolNames.push(tool.name); + } + agent.activateTools(toolNames); + log.info(`Skill ${skillName} chrome-devtools registered lazily for agent ${id}`); + } catch (error) { + log.warn(`Failed to register chrome-devtools lazily for agent ${id}`, { error: String(error) }); + } + })()); + continue; + } mcpConnections.push((async () => { try { let mcpTools: AgentToolHandler[]; @@ -1578,7 +1702,7 @@ export class AgentManager { await this.mcpManager.connectServerScoped(serverName, serverConfig, id); mcpTools = this.mcpManager.getToolHandlersScoped(serverName, id); mcpTools = this.browserSessionManager.wrapToolHandlers(mcpTools, id); - this.browserSessionManager.setReconnector(id, async () => { + this.browserSessionManager.setReconnector(id, serverName, async () => { await this.mcpManager.disconnectServerScoped(serverName, id); await this.mcpManager.connectServerScoped(serverName, serverConfig, id); }); @@ -1614,21 +1738,24 @@ export class AgentManager { const skill = this.skillRegistry?.get(skillName); const isolation = skill?.manifest.isolation ?? 'shared'; for (const [serverName, rawSrvConfig] of Object.entries(mcpServers)) { - if (isolation === 'per-agent' || isolation === 'pooled') { - const srvConfig = this.enrichChromeDevtoolsConfig(serverName, rawSrvConfig); + const srvConfig = this.enrichChromeDevtoolsConfig(serverName, rawSrvConfig); + if (serverName === 'chrome-devtools') { + const chromeTools = await this.registerChromeDevtoolsLazy(id, serverName, srvConfig); + tools.push(...chromeTools); + } else if (isolation === 'per-agent' || isolation === 'pooled') { await this.mcpManager.connectServerScoped(serverName, srvConfig, id); tools.push(...this.mcpManager.getToolHandlersScoped(serverName, id)); } else { - const srvConfig = this.enrichChromeDevtoolsConfig(serverName, rawSrvConfig); await this.mcpManager.connectServer(serverName, srvConfig); tools.push(...this.mcpManager.getToolHandlers(serverName)); } } - if (isolation === 'per-agent' || isolation === 'pooled') { + const hasChromeDevtools = Object.keys(mcpServers).includes('chrome-devtools'); + if (!hasChromeDevtools && (isolation === 'per-agent' || isolation === 'pooled')) { tools = this.browserSessionManager.wrapToolHandlers(tools, id); for (const [serverName, rawSrvConfig] of Object.entries(mcpServers)) { const srvConfig = this.enrichChromeDevtoolsConfig(serverName, rawSrvConfig); - this.browserSessionManager.setReconnector(id, async () => { + this.browserSessionManager.setReconnector(id, serverName, async () => { await this.mcpManager.disconnectServerScoped(serverName, id); await this.mcpManager.connectServerScoped(serverName, srvConfig, id); }); @@ -2234,8 +2361,9 @@ export class AgentManager { if (this.mcpReleaseTimers.has(agentId)) return; const timer = setTimeout(() => { this.mcpReleaseTimers.delete(agentId); - this.browserSessionManager.cleanupAgent(agentId); - this.mcpManager.disconnectAllForScope(agentId).catch(() => {}); + // Disconnect idle non-browser MCP processes. + // chrome-devtools manages its own lifecycle via the MCP idle timer (5min). + this.mcpManager.disconnectAllForScope(agentId, { skip: ['chrome-devtools'] }).catch(() => {}); log.info(`Released scoped MCP processes for idle agent: ${agentId}`); }, AgentManager.MCP_IDLE_GRACE_MS); timer.unref(); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 4ed01e3f..84be4669 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -235,3 +235,12 @@ export { type AgentDataProvider, type AgentDataRestorer, } from './agent-snapshot.js'; +export { + clickChromeAllowDialog, + checkAutoClickStatus, + testAutoClick, + type AutoClickCheckResult, + type AutoClickTestResult, +} from './tools/chrome-dialog-clicker.js'; +export { MarkusBrowserBridge } from './tools/markus-browser-bridge.js'; +export { createBridgeToolHandlers, getBridgeToolDescriptors } from './tools/markus-browser-mcp.js'; diff --git a/packages/core/src/tools/browser-session.ts b/packages/core/src/tools/browser-session.ts index 108ae883..c628f6f6 100644 --- a/packages/core/src/tools/browser-session.ts +++ b/packages/core/src/tools/browser-session.ts @@ -48,7 +48,7 @@ export class BrowserSessionManager { /** * Per-agent mutex: only one browser operation runs at a time per MCP process. * This prevents session A's "select → operate" from being interleaved with - * session B's operations. + * session B's operations within the same agent. */ private agentLocks = new Map>(); @@ -58,6 +58,7 @@ export class BrowserSessionManager { * going through the ownership check wrapper. */ private selectPageHandlers = new Map(); + private listPageHandlers = new Map(); /** * Per-agent callback to disconnect + reconnect the MCP server process. @@ -66,7 +67,7 @@ export class BrowserSessionManager { * Since handlers look up the server by key dynamically, they automatically * route to the new process after reconnect. */ - private reconnectors = new Map Promise>(); + private reconnectors = new Map Promise>>(); private _bringToFront = false; private _autoCloseTabs = true; @@ -77,11 +78,16 @@ export class BrowserSessionManager { set autoCloseTabs(v: boolean) { this._autoCloseTabs = v; } /** - * Register a reconnect callback for an agent's MCP server. - * Called by agent-manager after wrapToolHandlers. + * Register a reconnect callback for a specific MCP server of an agent. + * Multiple servers can each have their own reconnector without overwriting. */ - setReconnector(agentId: string, callback: () => Promise): void { - this.reconnectors.set(agentId, callback); + setReconnector(agentId: string, serverKey: string, callback: () => Promise): void { + let map = this.reconnectors.get(agentId); + if (!map) { + map = new Map(); + this.reconnectors.set(agentId, map); + } + map.set(serverKey, callback); } // ─── Stale page recovery ────────────────────────────────────────────────── @@ -91,11 +97,13 @@ export class BrowserSessionManager { } /** - * Clear all ownership state for an agent and reconnect its MCP server. - * Returns true if reconnect succeeded. + * Reconnect the MCP server for an agent. Preserves ownership state + * because page IDs remain stable across reconnects (Chrome stays running). + * Only clears the currentPage pointer so the next operation re-selects. */ private async reconnectMcp(agentId: string): Promise { - const reconnect = this.reconnectors.get(agentId); + const map = this.reconnectors.get(agentId); + const reconnect = map?.get('chrome-devtools'); if (!reconnect) { log.warn(`No reconnector available for agent ${agentId}`); return false; @@ -103,11 +111,11 @@ export class BrowserSessionManager { log.info(`Stale page detected for agent ${agentId} — reconnecting MCP server`); - // Clear all session state for this agent + // Only clear currentPage and lastActive (forces re-select on next op). + // Ownership is preserved — page IDs are stable since Chrome is still running. const prefix = `${agentId}::`; - for (const key of [...this.ownedPages.keys()]) { + for (const key of [...this.currentPage.keys()]) { if (key === agentId || key.startsWith(prefix)) { - this.ownedPages.delete(key); this.currentPage.delete(key); } } @@ -116,6 +124,7 @@ export class BrowserSessionManager { try { await reconnect(); log.info(`MCP server reconnected for agent ${agentId}`); + await this.pruneOwnedPages(agentId); return true; } catch (err) { log.error(`Failed to reconnect MCP server for agent ${agentId}: ${err}`); @@ -123,6 +132,35 @@ export class BrowserSessionManager { } } + /** + * After reconnect, call list_pages and prune ownedPages to only valid IDs. + * Pages that were closed while the MCP was disconnected are removed. + */ + private async pruneOwnedPages(agentId: string): Promise { + const listHandler = this.listPageHandlers.get(agentId); + if (!listHandler) return; + + try { + const result = await listHandler.execute({}); + const livePages = this.parsePageEntries(result); + const liveIds = new Set(livePages.map(p => p.id)); + + const prefix = `${agentId}::`; + for (const [key, owned] of this.ownedPages) { + if (key === agentId || key.startsWith(prefix)) { + for (const pageId of owned) { + if (!liveIds.has(pageId)) { + owned.delete(pageId); + log.debug(`Pruned stale page ${pageId} from ${key}`); + } + } + } + } + } catch (err) { + log.warn(`Failed to prune owned pages for ${agentId}: ${err}`); + } + } + // ─── Internal helpers ───────────────────────────────────────────────────── private async withAgentLock(agentId: string, fn: () => Promise): Promise { @@ -216,15 +254,20 @@ export class BrowserSessionManager { handlers.find((h) => (h.name.split('__').pop() ?? h.name) === name); const newPageHandler = findHandler('new_page'); const selectPageHandler = findHandler('select_page'); + const listPageHandler = findHandler('list_pages'); if (selectPageHandler) { this.selectPageHandlers.set(agentId, selectPageHandler); } + if (listPageHandler) { + this.listPageHandlers.set(agentId, listPageHandler); + } return handlers.map((h) => { const baseName = h.name.split('__').pop() ?? h.name; switch (baseName) { case 'new_page': + case 'open_page': return this.wrapNewPage(h, agentId); case 'list_pages': return this.wrapListPages(h, agentId); @@ -250,6 +293,9 @@ export class BrowserSessionManager { if (args.background === undefined) { args.background = !this._bringToFront; } + if (args.timeout === undefined && args.url) { + args.timeout = 60000; + } return this.withAgentLock(agentId, async () => { const owned = this.getOwned(ownerKey); const prevIds = new Set(owned); @@ -290,14 +336,16 @@ export class BrowserSessionManager { ...handler, execute: async (args: Record) => { const ownerKey = this.extractOwnerKey(agentId, args); - let result = await handler.execute(args); + return this.withAgentLock(agentId, async () => { + let result = await handler.execute(args); - if (this.isStalePageError(result)) { - const ok = await this.reconnectMcp(agentId); - if (ok) result = await handler.execute(args); - } + if (this.isStalePageError(result)) { + const ok = await this.reconnectMcp(agentId); + if (ok) result = await handler.execute(args); + } - return this.annotateResponse(result, ownerKey); + return this.annotateResponse(result, ownerKey); + }); }, }; } @@ -322,8 +370,15 @@ export class BrowserSessionManager { if (this.isStalePageError(result)) { const ok = await this.reconnectMcp(agentId); if (ok) { - return 'Browser session was reset because tabs were closed externally. ' - + 'Your previously owned tabs are gone. Call new_page or navigate_page to create a new tab.'; + // Remove only the failed page from ownership + if (pageId !== undefined) { + this.getOwned(ownerKey).delete(pageId); + if (this.currentPage.get(ownerKey) === pageId) { + this.currentPage.delete(ownerKey); + } + } + return `Tab ${pageId ?? 'unknown'} was closed externally. ` + + `${this.ownedPagesSummary(ownerKey)} Call new_page or navigate_page to create a new tab.`; } } @@ -354,8 +409,17 @@ export class BrowserSessionManager { if (this.isStalePageError(result)) { const ok = await this.reconnectMcp(agentId); if (ok) { - return 'Browser session was reset because tabs were closed externally. ' - + 'Your previously owned tabs are gone. Call new_page or navigate_page to create a new tab.'; + // Remove only the failed page from ownership + if (pageId !== undefined) { + this.getOwned(ownerKey).delete(pageId); + } + const remaining = this.getOwned(ownerKey); + if (remaining.size > 0) { + return `Tab ${pageId ?? 'unknown'} was already closed externally. ` + + `${this.ownedPagesSummary(ownerKey)}`; + } + return `Tab ${pageId ?? 'unknown'} was closed externally and you have no remaining tabs. ` + + 'Call new_page or navigate_page to create a new tab.'; } } @@ -385,6 +449,9 @@ export class BrowserSessionManager { return { ...handler, execute: async (args: Record) => { + if (args.timeout === undefined) { + args.timeout = 60000; + } const ownerKey = this.extractOwnerKey(agentId, args); const owned = this.getOwned(ownerKey); @@ -476,6 +543,9 @@ export class BrowserSessionManager { return { ...handler, execute: async (args: Record) => { + if (args.timeout === undefined && args.url) { + args.timeout = 60000; + } const ownerKey = this.extractOwnerKey(agentId, args); const owned = this.getOwned(ownerKey); if (owned.size === 0) { @@ -496,8 +566,14 @@ export class BrowserSessionManager { if (this.isStalePageError(result)) { const ok = await this.reconnectMcp(agentId); if (ok) { - return 'Browser session was reset because tabs were closed externally. ' - + 'Your previously owned tabs are gone. Call navigate_page or new_page to create a new tab, then retry.'; + // Remove the stale page from ownership + const currentPageId = this.currentPage.get(ownerKey); + if (currentPageId !== undefined) { + this.getOwned(ownerKey).delete(currentPageId); + this.currentPage.delete(ownerKey); + } + return `The tab you were operating on was closed externally. ` + + `${this.ownedPagesSummary(ownerKey)} Call navigate_page or new_page to create a new tab, then retry.`; } } @@ -521,6 +597,7 @@ export class BrowserSessionManager { } this.agentLocks.delete(agentId); this.selectPageHandlers.delete(agentId); + this.listPageHandlers.delete(agentId); this.lastActiveSession.delete(agentId); this.reconnectors.delete(agentId); if (total > 0) { diff --git a/packages/core/src/tools/chrome-dialog-clicker.ts b/packages/core/src/tools/chrome-dialog-clicker.ts new file mode 100644 index 00000000..69853344 --- /dev/null +++ b/packages/core/src/tools/chrome-dialog-clicker.ts @@ -0,0 +1,325 @@ +import { execFile, spawn } from 'node:child_process'; +import { platform } from 'node:os'; +import { resolve, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { existsSync } from 'node:fs'; +import { createLogger } from '@markus/shared'; + +const log = createLogger('chrome-dialog-clicker'); + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +const SCRIPTS_DIR = resolve(__dirname, '../../../../scripts/markus-chrome-allow'); + +export interface AutoClickCheckResult { + platform: 'darwin' | 'win32' | 'linux' | string; + supported: boolean; + accessibilityPermission: boolean; + chromeRunning: boolean; + binaryAvailable: boolean; +} + +export interface AutoClickTestResult { + checkResult: AutoClickCheckResult; + openedAccessibilitySettings: boolean; + clickResult: 'success' | 'no_permission' | 'chrome_not_running' | 'unsupported' | 'error'; + pageLoaded: boolean; + pageTitle?: string; + error?: string; +} + +/** + * Check current platform's readiness for auto-click (permissions, Chrome status). + */ +export async function checkAutoClickStatus(): Promise { + const os = platform(); + const base: AutoClickCheckResult = { + platform: os, + supported: os === 'darwin' || os === 'win32', + accessibilityPermission: false, + chromeRunning: false, + binaryAvailable: false, + }; + + if (os === 'darwin') { + const bin = resolve(SCRIPTS_DIR, 'markus-chrome-allow'); + base.binaryAvailable = existsSync(bin); + if (!base.binaryAvailable) return base; + try { + const result = await runHelperRaw(bin, ['--check'], 5); + const data = JSON.parse(result); + base.accessibilityPermission = data.accessibilityPermission === true; + base.chromeRunning = data.chromeRunning === true; + } catch { /* keep defaults */ } + return base; + } + + if (os === 'win32') { + const script = resolve(SCRIPTS_DIR, 'markus-chrome-allow.ps1'); + base.binaryAvailable = existsSync(script); + if (!base.binaryAvailable) return base; + try { + const result = await runHelperRaw( + 'powershell', + ['-ExecutionPolicy', 'Bypass', '-File', script, '-Check'], + 5, + ); + const data = JSON.parse(result); + base.accessibilityPermission = data.accessibilityPermission === true; + base.chromeRunning = data.chromeRunning === true; + } catch { /* keep defaults */ } + return base; + } + + return base; +} + +/** + * Open the macOS Accessibility settings pane. + */ +export async function openAccessibilitySettings(): Promise { + const os = platform(); + if (os === 'darwin') { + const bin = resolve(SCRIPTS_DIR, 'markus-chrome-allow'); + if (!existsSync(bin)) return false; + try { + await runHelperRaw(bin, ['--open-accessibility'], 3); + return true; + } catch { return false; } + } + return false; +} + +/** + * Run a full end-to-end test: + * 1. Check permissions and Chrome status + * 2. If no permission on macOS, open settings page + * 3. Spawn a temporary chrome-devtools-mcp connection (triggers the dialog) + * 4. Auto-click the "Allow" dialog + * 5. Navigate to a test URL and verify page loaded + * 6. Clean up and return results + */ +export async function testAutoClick(): Promise { + const checkResult = await checkAutoClickStatus(); + const result: AutoClickTestResult = { + checkResult, + openedAccessibilitySettings: false, + clickResult: 'unsupported', + pageLoaded: false, + }; + + if (!checkResult.supported) { + result.clickResult = 'unsupported'; + return result; + } + + if (!checkResult.binaryAvailable) { + result.clickResult = 'error'; + result.error = 'Helper binary not found'; + return result; + } + + if (platform() === 'darwin' && !checkResult.accessibilityPermission) { + result.openedAccessibilitySettings = await openAccessibilitySettings(); + result.clickResult = 'no_permission'; + return result; + } + + if (!checkResult.chromeRunning) { + result.clickResult = 'chrome_not_running'; + return result; + } + + // Spawn a temporary chrome-devtools-mcp to trigger the Allow dialog + navigate + try { + const { navigated, title } = await runMcpTest(); + result.pageLoaded = navigated; + result.pageTitle = title; + result.clickResult = 'success'; + } catch (err) { + const msg = String(err instanceof Error ? err.message : err); + if (msg.includes('Accessibility permission')) { + result.clickResult = 'no_permission'; + } else { + result.clickResult = 'error'; + result.error = msg.slice(0, 200); + } + } + + return result; +} + +/** + * Spawn chrome-devtools-mcp, auto-click the Allow dialog, then navigate to a page. + * If Chrome is already in debug mode, connection succeeds without dialog. + * Returns whether navigation succeeded and the page title. + */ +async function runMcpTest(): Promise<{ navigated: boolean; title?: string }> { + + const npxCmd = platform() === 'win32' ? 'npx.cmd' : 'npx'; + + return new Promise((resolveTest, rejectTest) => { + const stderrChunks: string[] = []; + const proc = spawn(npxCmd, ['-y', 'chrome-devtools-mcp@latest', '--autoConnect'], { + stdio: ['pipe', 'pipe', 'pipe'], + env: { ...process.env }, + shell: platform() === 'win32', + }); + + let stdout = ''; + let requestId = 1; + const pending = new Map void; reject: (e: Error) => void }>(); + let cleaned = false; + + const cleanup = () => { + if (cleaned) return; + cleaned = true; + try { proc.kill(); } catch { /* already dead */ } + }; + + const timeout = setTimeout(() => { + cleanup(); + const stderr = stderrChunks.join('').slice(0, 300); + rejectTest(new Error(`MCP test timed out after 60s${stderr ? ` (stderr: ${stderr})` : ''}`)); + }, 60000); + + proc.on('error', (err) => { + clearTimeout(timeout); + cleanup(); + rejectTest(err); + }); + + proc.stderr?.on('data', (data: Buffer) => { + stderrChunks.push(data.toString()); + }); + + proc.on('exit', (code) => { + clearTimeout(timeout); + if (!cleaned) { + cleaned = true; + const stderr = stderrChunks.join('').slice(0, 300); + rejectTest(new Error(`MCP process exited (code ${code})${stderr ? `: ${stderr}` : ''}`)); + } + }); + + proc.stdout?.on('data', (data: Buffer) => { + stdout += data.toString(); + const lines = stdout.split('\n'); + stdout = lines.pop() ?? ''; + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed) continue; + try { + const msg = JSON.parse(trimmed); + if (msg.id !== null && msg.id !== undefined && pending.has(msg.id)) { + const p = pending.get(msg.id)!; + pending.delete(msg.id); + if (msg.error) p.reject(new Error(msg.error.message ?? JSON.stringify(msg.error))); + else p.resolve(msg.result); + } + } catch { /* not JSON */ } + } + }); + + const sendRequest = (method: string, params: unknown): Promise => { + const id = requestId++; + const msg = JSON.stringify({ jsonrpc: '2.0', id, method, params }) + '\n'; + proc.stdin?.write(msg); + return new Promise((res, rej) => { pending.set(id, { resolve: res, reject: rej }); }); + }; + + const sendNotification = (method: string, params: unknown) => { + const msg = JSON.stringify({ jsonrpc: '2.0', method, params }) + '\n'; + proc.stdin?.write(msg); + }; + + // Fire auto-clicker with generous timeout — dialog only appears after npx + // finishes downloading and the MCP server attempts its Chrome connection. + clickChromeAllowDialog(30).catch(() => {}); + + // Run the MCP handshake + navigate sequence + (async () => { + try { + await sendRequest('initialize', { + protocolVersion: '2024-11-05', + capabilities: {}, + clientInfo: { name: 'markus-test', version: '1.0.0' }, + }); + sendNotification('notifications/initialized', {}); + await sendRequest('tools/list', {}); + + // open_page triggers Chrome CDP connection → "Allow" dialog + // Open in foreground so the user can see the test working + const navResult = await sendRequest('tools/call', { + name: 'open_page', + arguments: { url: 'https://example.com' }, + }) as { content?: Array<{ text?: string }> }; + + const text = navResult?.content?.[0]?.text ?? ''; + const navigated = !text.toLowerCase().includes('error'); + + clearTimeout(timeout); + cleanup(); + resolveTest({ navigated, title: navigated ? 'example.com' : undefined }); + } catch (err) { + clearTimeout(timeout); + cleanup(); + rejectTest(err instanceof Error ? err : new Error(String(err))); + } + })(); + }); +} + +/** + * Auto-click Chrome's "Allow remote debugging?" dialog. + */ +export async function clickChromeAllowDialog(timeoutSec = 5): Promise { + const os = platform(); + + if (os === 'darwin') { + const bin = resolve(SCRIPTS_DIR, 'markus-chrome-allow'); + return runHelper(bin, ['--timeout', String(timeoutSec)], timeoutSec); + } + if (os === 'win32') { + const script = resolve(SCRIPTS_DIR, 'markus-chrome-allow.ps1'); + return runHelper( + 'powershell', + ['-ExecutionPolicy', 'Bypass', '-File', script, '-Timeout', String(timeoutSec)], + timeoutSec, + ); + } + + log.debug('Auto-click Chrome dialog not supported on this platform'); + return false; +} + +function runHelper(cmd: string, args: string[], timeoutSec: number): Promise { + return new Promise((resolve) => { + execFile(cmd, args, { timeout: (timeoutSec + 2) * 1000 }, (err, stdout) => { + if (err) { + log.debug('Chrome dialog clicker failed', { error: String(err) }); + resolve(false); + return; + } + try { + const r = JSON.parse(stdout); + if (r.clicked === true) { + log.info('Auto-clicked Chrome "Allow debugging" dialog'); + } + resolve(r.clicked === true); + } catch { + resolve(false); + } + }); + }); +} + +function runHelperRaw(cmd: string, args: string[], timeoutSec: number): Promise { + return new Promise((resolve, reject) => { + execFile(cmd, args, { timeout: (timeoutSec + 2) * 1000 }, (err, stdout) => { + if (err) { reject(err); return; } + resolve(stdout.trim()); + }); + }); +} diff --git a/packages/core/src/tools/markus-browser-bridge.ts b/packages/core/src/tools/markus-browser-bridge.ts new file mode 100644 index 00000000..23ba0b73 --- /dev/null +++ b/packages/core/src/tools/markus-browser-bridge.ts @@ -0,0 +1,164 @@ +/** + * WebSocket bridge between Markus and the Chrome extension. + * + * Markus side (this file) runs a WS server. The Chrome extension connects + * as a client. Tool calls are forwarded over the socket and results returned. + */ + +import { WebSocketServer, type WebSocket } from 'ws'; +import { createLogger } from '@markus/shared'; + +const log = createLogger('browser-bridge'); + +export interface BridgeToolResult { + content: string; + error?: string; +} + +interface PendingCall { + resolve: (result: BridgeToolResult) => void; + reject: (err: Error) => void; + timer: ReturnType; +} + +const DEFAULT_PORT = 9333; +const TOOL_CALL_TIMEOUT_MS = 120_000; + +export class MarkusBrowserBridge { + private wss: WebSocketServer | null = null; + private client: WebSocket | null = null; + private requestId = 0; + private pending = new Map(); + private port: number; + private _started = false; + private connectionListeners: Array<(connected: boolean) => void> = []; + + constructor(port?: number) { + this.port = port ?? DEFAULT_PORT; + } + + get started(): boolean { return this._started; } + get connected(): boolean { return this.client?.readyState === 1; } + + onConnectionChange(listener: (connected: boolean) => void): void { + this.connectionListeners.push(listener); + } + + private notifyConnectionChange(connected: boolean): void { + for (const listener of this.connectionListeners) { + try { listener(connected); } catch { /* ignore */ } + } + } + + start(): void { + if (this._started) return; + this._started = true; + + this.wss = new WebSocketServer({ port: this.port, host: '127.0.0.1' }); + + this.wss.on('listening', () => { + log.info(`Browser bridge WebSocket server listening on ws://127.0.0.1:${this.port}`); + }); + + this.wss.on('error', (err) => { + log.warn(`Browser bridge WebSocket server error: ${err.message}`); + }); + + this.wss.on('connection', (ws) => { + if (this.client) { + log.info('New extension connection replacing existing one'); + this.client.close(); + } + this.client = ws; + log.info('Chrome extension connected to browser bridge'); + this.notifyConnectionChange(true); + + ws.on('message', (data) => { + try { + const msg = JSON.parse(data.toString()); + this.handleMessage(msg); + } catch (err) { + log.warn(`Invalid message from extension: ${err}`); + } + }); + + ws.on('close', () => { + if (this.client === ws) { + this.client = null; + log.info('Chrome extension disconnected from browser bridge'); + this.notifyConnectionChange(false); + this.rejectAllPending('Extension disconnected'); + } + }); + + ws.on('error', (err) => { + log.warn(`Extension WebSocket error: ${err.message}`); + }); + }); + } + + stop(): void { + this.rejectAllPending('Bridge shutting down'); + if (this.client) { + this.client.close(); + this.client = null; + } + if (this.wss) { + this.wss.close(); + this.wss = null; + } + this._started = false; + } + + /** + * Call a tool on the Chrome extension and wait for the result. + */ + async callTool(name: string, args: Record): Promise { + if (!this.connected) { + throw new Error('Chrome extension not connected'); + } + + const id = ++this.requestId; + const message = JSON.stringify({ id, method: name, params: args }); + + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.pending.delete(id); + reject(new Error(`Tool call ${name} timed out after ${TOOL_CALL_TIMEOUT_MS}ms`)); + }, TOOL_CALL_TIMEOUT_MS); + + this.pending.set(id, { resolve, reject, timer }); + this.client!.send(message); + }); + } + + private handleMessage(msg: { id?: number; result?: unknown; error?: string; event?: string; data?: unknown }): void { + if (msg.event) { + log.debug(`Extension event: ${msg.event}`, msg.data as Record); + return; + } + + if (msg.id === undefined) return; + + const pending = this.pending.get(msg.id); + if (!pending) return; + + this.pending.delete(msg.id); + clearTimeout(pending.timer); + + if (msg.error) { + pending.resolve({ content: '', error: msg.error }); + } else { + const text = typeof msg.result === 'string' ? msg.result : JSON.stringify(msg.result); + pending.resolve({ content: text }); + } + } + + private rejectAllPending(reason: string): void { + for (const [id, pending] of this.pending) { + clearTimeout(pending.timer); + pending.reject(new Error(reason)); + this.pending.delete(id); + } + } +} diff --git a/packages/core/src/tools/markus-browser-mcp.ts b/packages/core/src/tools/markus-browser-mcp.ts new file mode 100644 index 00000000..1ddd6b4b --- /dev/null +++ b/packages/core/src/tools/markus-browser-mcp.ts @@ -0,0 +1,190 @@ +/** + * In-process MCP-compatible tool provider that forwards tool calls + * to the Chrome extension via the WebSocket bridge. + * + * When the extension is connected, this replaces chrome-devtools-mcp entirely, + * avoiding the "Allow debugging?" dialog and npx startup overhead. + * + * The tool list mirrors chrome-devtools-mcp's tools so that BrowserSessionManager + * and agents work identically regardless of which backend is used. + */ + +import { createLogger } from '@markus/shared'; +import type { MarkusBrowserBridge } from './markus-browser-bridge.js'; +import type { MCPToolDescriptor } from './mcp-client.js'; +import type { AgentToolHandler } from '../agent.js'; + +const log = createLogger('browser-mcp'); + +/** + * Tool descriptors matching chrome-devtools-mcp's tool interface. + * These are registered when the extension is connected. + */ +const TOOL_DESCRIPTORS: MCPToolDescriptor[] = [ + // Navigation tools + { + name: 'new_page', + description: 'Create a new browser page/tab and optionally navigate to a URL', + inputSchema: { type: 'object', properties: { url: { type: 'string', description: 'URL to navigate to (default: about:blank)' }, background: { type: 'boolean', description: 'Open in background' }, timeout: { type: 'number', description: 'Timeout in ms (default: 60000)' } } }, + }, + { + name: 'open_page', + description: 'Open a new browser page/tab and navigate to a URL', + inputSchema: { type: 'object', properties: { url: { type: 'string', description: 'URL to navigate to' }, background: { type: 'boolean', description: 'Open in background' }, timeout: { type: 'number', description: 'Timeout in ms (default: 60000)' } } }, + }, + { + name: 'close_page', + description: 'Close a browser page/tab', + inputSchema: { type: 'object', properties: { pageId: { type: 'number', description: 'Page ID to close' } }, required: ['pageId'] }, + }, + { + name: 'list_pages', + description: 'List all open browser pages/tabs', + inputSchema: { type: 'object', properties: {} }, + }, + { + name: 'select_page', + description: 'Select a browser page/tab as the active one', + inputSchema: { type: 'object', properties: { pageId: { type: 'number', description: 'Page ID to select' } }, required: ['pageId'] }, + }, + { + name: 'navigate_page', + description: 'Navigate the selected page to a URL or go back/forward/reload', + inputSchema: { type: 'object', properties: { url: { type: 'string', description: 'URL to navigate to' }, action: { type: 'string', enum: ['back', 'forward', 'reload'] }, timeout: { type: 'number', description: 'Timeout in ms' } } }, + }, + { + name: 'wait_for', + description: 'Wait for text to appear on the selected page', + inputSchema: { type: 'object', properties: { text: { type: 'string', description: 'Text to wait for' }, timeout: { type: 'number', description: 'Timeout in ms (default: 30000)' } }, required: ['text'] }, + }, + // Input tools + { + name: 'click', + description: 'Click an element identified by its accessibility snapshot uid', + inputSchema: { type: 'object', properties: { uid: { type: 'string', description: 'Element uid from snapshot' } }, required: ['uid'] }, + }, + { + name: 'fill', + description: 'Fill an input element with text (replaces existing content)', + inputSchema: { type: 'object', properties: { uid: { type: 'string', description: 'Element uid from snapshot' }, value: { type: 'string', description: 'Text to fill' } }, required: ['uid', 'value'] }, + }, + { + name: 'fill_form', + description: 'Fill multiple form fields at once', + inputSchema: { type: 'object', properties: { fields: { type: 'array', items: { type: 'object', properties: { uid: { type: 'string' }, value: { type: 'string' } }, required: ['uid', 'value'] } } }, required: ['fields'] }, + }, + { + name: 'type_text', + description: 'Type text into the currently focused element', + inputSchema: { type: 'object', properties: { text: { type: 'string', description: 'Text to type' } }, required: ['text'] }, + }, + { + name: 'press_key', + description: 'Press a keyboard key or key combination (e.g. Enter, Ctrl+A)', + inputSchema: { type: 'object', properties: { key: { type: 'string', description: 'Key or combination (e.g. Enter, Ctrl+A)' } }, required: ['key'] }, + }, + { + name: 'hover', + description: 'Hover over an element identified by its accessibility snapshot uid', + inputSchema: { type: 'object', properties: { uid: { type: 'string', description: 'Element uid from snapshot' } }, required: ['uid'] }, + }, + { + name: 'drag', + description: 'Drag from one element to another', + inputSchema: { type: 'object', properties: { from_uid: { type: 'string' }, to_uid: { type: 'string' } }, required: ['from_uid', 'to_uid'] }, + }, + { + name: 'handle_dialog', + description: 'Accept or dismiss a JavaScript dialog (alert, confirm, prompt)', + inputSchema: { type: 'object', properties: { accept: { type: 'boolean', description: 'Accept (true) or dismiss (false)' }, promptText: { type: 'string', description: 'Text for prompt dialog' } } }, + }, + { + name: 'upload_file', + description: 'Upload a file to a file input element', + inputSchema: { type: 'object', properties: { uid: { type: 'string', description: 'File input element uid' }, filePath: { type: 'string', description: 'Path to file to upload' } }, required: ['uid', 'filePath'] }, + }, + // Inspection tools + { + name: 'take_screenshot', + description: 'Take a screenshot of the selected page', + inputSchema: { type: 'object', properties: { format: { type: 'string', description: 'Image format (png or jpg)' }, quality: { type: 'number', description: 'Quality 0-100 for jpg' }, fullPage: { type: 'boolean', description: 'Capture full page' } } }, + }, + { + name: 'take_snapshot', + description: 'Take an accessibility tree snapshot of the selected page', + inputSchema: { type: 'object', properties: {} }, + }, + { + name: 'evaluate_script', + description: 'Evaluate JavaScript in the selected page', + inputSchema: { type: 'object', properties: { expression: { type: 'string', description: 'JavaScript expression to evaluate' } }, required: ['expression'] }, + }, + { + name: 'get_console_message', + description: 'Get a specific console message by ID', + inputSchema: { type: 'object', properties: { msgid: { type: 'number', description: 'Message ID' } }, required: ['msgid'] }, + }, + { + name: 'list_console_messages', + description: 'List all console messages from the selected page', + inputSchema: { type: 'object', properties: {} }, + }, + { + name: 'lighthouse_audit', + description: 'Run an accessibility audit on the selected page', + inputSchema: { type: 'object', properties: { categories: { type: 'array', items: { type: 'string' } } } }, + }, + // Network tools + { + name: 'list_network_requests', + description: 'List captured network requests from the selected page', + inputSchema: { type: 'object', properties: {} }, + }, + { + name: 'get_network_request', + description: 'Get details of a specific network request', + inputSchema: { type: 'object', properties: { reqid: { type: 'string', description: 'Request ID' } }, required: ['reqid'] }, + }, + // Emulation tools + { + name: 'emulate', + description: 'Set device emulation (viewport, user agent, geolocation, color scheme)', + inputSchema: { type: 'object', properties: { width: { type: 'number' }, height: { type: 'number' }, deviceScaleFactor: { type: 'number' }, mobile: { type: 'boolean' }, userAgent: { type: 'string' }, geolocation: { type: 'object', properties: { latitude: { type: 'number' }, longitude: { type: 'number' } } }, colorScheme: { type: 'string' } } }, + }, + { + name: 'resize_page', + description: 'Resize the browser window', + inputSchema: { type: 'object', properties: { width: { type: 'number' }, height: { type: 'number' } }, required: ['width', 'height'] }, + }, +]; + +/** + * Creates tool handlers that forward calls through the browser bridge. + * These handlers have the same interface as MCPClientManager's tool handlers, + * so they can be used with BrowserSessionManager.wrapToolHandlers(). + */ +export function createBridgeToolHandlers( + bridge: MarkusBrowserBridge, + serverName: string, +): AgentToolHandler[] { + return TOOL_DESCRIPTORS.map((tool) => ({ + name: `${serverName}__${tool.name}`, + description: `[MCP:${serverName}] ${tool.description}`, + inputSchema: tool.inputSchema, + execute: async (args: Record) => { + const result = await bridge.callTool(tool.name, args); + if (result.error) { + return `Error: ${result.error}`; + } + return result.content; + }, + })); +} + +/** + * Get the static tool descriptors (useful for lazy registration when + * the extension is connected but we don't need to spawn chrome-devtools-mcp). + */ +export function getBridgeToolDescriptors(): MCPToolDescriptor[] { + return TOOL_DESCRIPTORS; +} diff --git a/packages/core/src/tools/mcp-client.ts b/packages/core/src/tools/mcp-client.ts index 695fc1c3..8990eadc 100644 --- a/packages/core/src/tools/mcp-client.ts +++ b/packages/core/src/tools/mcp-client.ts @@ -10,7 +10,7 @@ interface MCPServerConfig { env?: Record; } -interface MCPToolDescriptor { +export interface MCPToolDescriptor { name: string; description: string; inputSchema: Record; @@ -48,11 +48,16 @@ export class MCPClientManager { private stdoutBuffers = new Map(); private idleTimers = new Map>(); private idleTimeoutMs: number = DEFAULT_IDLE_TIMEOUT_MS; + private onReconnectCallback?: (serverName: string) => void; private static scopedKey(name: string, scopeId: string): string { return `${name}::${scopeId}`; } + setOnReconnect(callback: (serverName: string) => void): void { + this.onReconnectCallback = callback; + } + setIdleTimeout(ms: number): void { this.idleTimeoutMs = ms; } @@ -150,6 +155,9 @@ export class MCPClientManager { this.servers.set(key, { process: proc, tools }); this.resetIdleTimer(key); + // Cache tool descriptors by base server name (for lazy registration) + const baseName = key.split('::')[0]; + this.toolCache.set(baseName, tools); log.info(`MCP server ${displayName} connected with ${tools.length} tools`); return tools; @@ -167,10 +175,35 @@ export class MCPClientManager { /** * Connect a scoped (per-agent) instance of the MCP server. * Each (name, scopeId) pair gets its own child process. + * Serialized via startup lock to prevent concurrent connections to the same external resource. */ async connectServerScoped(name: string, config: MCPServerConfig, scopeId: string): Promise { const key = MCPClientManager.scopedKey(name, scopeId); - return this.connectByKey(key, `${name}[${scopeId}]`, config); + let tools: MCPToolDescriptor[] = []; + await this.withStartupLock(name, async () => { + tools = await this.connectByKey(key, `${name}[${scopeId}]`, config); + }); + return tools; + } + + /** + * Startup semaphore: serializes MCP process creation for servers that share + * an external resource (e.g. chrome-devtools → Chrome). Prevents concurrent + * CDP connections from crashing Chrome. + */ + private startupLocks = new Map>(); + + private async withStartupLock(serverName: string, fn: () => Promise): Promise { + const prev = this.startupLocks.get(serverName) ?? Promise.resolve(); + let release: () => void; + const gate = new Promise(r => { release = r; }); + this.startupLocks.set(serverName, gate); + await prev; + try { + await fn(); + } finally { + release!(); + } } private async callToolByKey(key: string, toolName: string, args: Record): Promise { @@ -180,7 +213,13 @@ export class MCPClientManager { const saved = this.serverConfigs.get(key); if (saved) { log.info(`MCP server ${key} not running, auto-reconnecting...`); - await this.connectByKey(key, saved.displayName, saved.config); + const serverName = key.split('::')[0]; + this.onReconnectCallback?.(serverName); + await this.withStartupLock(serverName, async () => { + if (!this.servers.has(key)) { + await this.connectByKey(key, saved.displayName, saved.config); + } + }); server = this.servers.get(key); } if (!server) throw new Error(`MCP server not found: ${key}`); @@ -219,6 +258,26 @@ export class MCPClientManager { })); } + /** + * Register a server config and tool descriptors WITHOUT starting the process. + * Returns tool handlers that will auto-connect on first call via callToolByKey. + * Used for lazy-start servers (e.g. chrome-devtools: only connect when agent + * actually calls a browser tool). + */ + registerLazyScoped(name: string, config: MCPServerConfig, scopeId: string, tools: MCPToolDescriptor[]): AgentToolHandler[] { + const key = MCPClientManager.scopedKey(name, scopeId); + const displayName = `${name}[${scopeId}]`; + this.serverConfigs.set(key, { displayName, config }); + return tools.map((tool) => ({ + name: `${name}__${tool.name}`, + description: `[MCP:${name}] ${tool.description}`, + inputSchema: tool.inputSchema, + execute: async (args: Record) => { + return this.callToolByKey(key, tool.name, args); + }, + })); + } + getToolHandlers(serverName: string): AgentToolHandler[] { return this.getToolHandlersByKey(serverName, serverName); } @@ -239,6 +298,21 @@ export class MCPClientManager { })); } + /** + * Get cached tool descriptors for a server (from any scoped or shared instance). + * Returns undefined if no instance has connected yet. + */ + getCachedTools(serverName: string): MCPToolDescriptor[] | undefined { + for (const [key, server] of this.servers) { + if (key === serverName || key.startsWith(`${serverName}::`)) { + return server.tools; + } + } + return this.toolCache.get(serverName); + } + + private toolCache = new Map(); + async disconnectServer(name: string): Promise { const server = this.servers.get(name); if (server) { @@ -260,10 +334,13 @@ export class MCPClientManager { * Shared (non-scoped) servers are not affected. * The server configs are retained so the server can auto-reconnect on next tool call. */ - async disconnectAllForScope(scopeId: string): Promise { + async disconnectAllForScope(scopeId: string, opts?: { skip?: string[] }): Promise { const suffix = `::${scopeId}`; + const skipSet = opts?.skip ? new Set(opts.skip) : undefined; for (const key of [...this.servers.keys()]) { if (key.endsWith(suffix)) { + const serverName = key.split('::')[0]; + if (skipSet?.has(serverName)) continue; await this.disconnectServer(key); } } @@ -330,7 +407,7 @@ export class MCPClientManager { const id = ++this.requestId; const message = JSON.stringify({ jsonrpc: '2.0', id, method, params }) + '\n'; - const timeoutMs = method === 'tools/call' ? 120_000 : 30_000; + const timeoutMs = method === 'tools/call' ? 120_000 : 60_000; const timer = setTimeout(() => { this.pendingRequests.delete(id); reject(new Error(`MCP request timeout for ${method} (id=${id}, ${timeoutMs}ms)`)); diff --git a/packages/org-manager/src/api-server.ts b/packages/org-manager/src/api-server.ts index cf494656..7939f54a 100644 --- a/packages/org-manager/src/api-server.ts +++ b/packages/org-manager/src/api-server.ts @@ -197,6 +197,8 @@ export class APIServer { private workflowEngine?: WorkflowEngine; private teamTemplateRegistry: TeamTemplateRegistry; private fileStorage?: LocalFileStorageProvider; + private remoteAgent?: { getStatus(): unknown; start(): Promise; stop(): Promise; onStatus(cb: (s: unknown) => void): () => void }; + private remoteAgentFactory?: () => Promise<{ getStatus(): unknown; start(): Promise; stop(): Promise; onStatus(cb: (s: unknown) => void): () => void } | null>; // Custom group chats are now persisted in SQLite via storage.groupChatRepo constructor( private orgService: OrganizationService, @@ -268,6 +270,7 @@ export class APIServer { replyToId, }); persistedMsgId = saved.id; + this.ws.broadcastUnreadUpdate(`channel:${channelKey}`, saved.id); } // Send to frontend via WebSocket (scoped to channel members) @@ -526,6 +529,17 @@ export class APIServer { this.storage = storage; } + setRemoteAgent(agent: { getStatus(): unknown; start(): Promise; stop(): Promise; onStatus(cb: (s: unknown) => void): () => void }): void { + this.remoteAgent = agent; + agent.onStatus((status) => { + this.ws.broadcast({ type: 'remote:status', payload: status, timestamp: new Date().toISOString() }); + }); + } + + setRemoteAgentFactory(factory: () => Promise<{ getStatus(): unknown; start(): Promise; stop(): Promise; onStatus(cb: (s: unknown) => void): () => void } | null>): void { + this.remoteAgentFactory = factory; + } + setGateway(gateway: ExternalAgentGateway, secret?: string): void { this.gateway = gateway; this.gatewaySecret = secret; @@ -1317,7 +1331,7 @@ export class APIServer { ): Promise { if (!this.storage || !sessionId) return; try { - await this.storage.chatSessionRepo.appendMessage( + const msg = await this.storage.chatSessionRepo.appendMessage( sessionId, agentId, 'assistant', @@ -1326,6 +1340,9 @@ export class APIServer { metadata ); await this.storage.chatSessionRepo.updateLastMessage(sessionId); + if (msg?.id) { + this.ws.broadcastUnreadUpdate(`session:${sessionId}`, msg.id); + } } catch (err) { log.warn('Failed to persist assistant message', { error: String(err) }); } @@ -1983,6 +2000,11 @@ export class APIServer { }); } + // Broadcast unread update for channel message + if (userMsg) { + this.ws.broadcastUnreadUpdate(`channel:${channel}`, userMsg.id); + } + // DM / personal-notepad channels never route to agents const humanOnly = (body['humanOnly'] as boolean) === true; const isHumanChannel = humanOnly || channel.startsWith('notes:') || channel.startsWith('dm:'); @@ -3067,6 +3089,21 @@ export class APIServer { return; } + if (path.match(/^\/api\/deliverables\/[^/]+$/) && req.method === 'GET') { + const authUser = await this.requireAuth(req, res); + if (!authUser) return; + if (!this.deliverableService) { this.json(res, 503, { error: 'Deliverable service not available' }); return; } + const delivId = path.split('/')[3]!; + try { + const d = await this.deliverableService.get(delivId); + if (!d) { this.json(res, 404, { error: 'Deliverable not found' }); return; } + this.json(res, 200, { deliverable: d }); + } catch (err) { + this.json(res, 500, { error: String(err) }); + } + return; + } + if (path.match(/^\/api\/deliverables\/[^/]+$/) && req.method === 'PUT') { const authUser = await this.requireAuth(req, res); if (!authUser) return; @@ -6494,6 +6531,40 @@ EXPLANATION_END`; return; } + // ── Unread message tracking ────────────────────────────────────────────── + if (path === '/api/unread' && req.method === 'GET') { + const authUser = await this.requireAuth(req, res); + if (!authUser) return; + const repo = this.storage?.readCursorRepo; + if (!repo) { this.json(res, 200, { counts: {} }); return; } + const counts = repo.getUnreadCounts(authUser.userId); + this.json(res, 200, { counts }); + return; + } + + if (path === '/api/unread/mark-read' && req.method === 'POST') { + const authUser = await this.requireAuth(req, res); + if (!authUser) return; + const body = await this.readBody(req); + const { conversationKey, lastReadAt, lastReadId } = body as { conversationKey: string; lastReadAt: string; lastReadId?: string }; + if (!conversationKey || !lastReadAt) { this.json(res, 400, { error: 'conversationKey and lastReadAt required' }); return; } + const repo = this.storage?.readCursorRepo; + if (!repo) { this.json(res, 200, { success: true }); return; } + repo.setReadCursor(authUser.userId, conversationKey, lastReadAt, lastReadId); + this.json(res, 200, { success: true }); + return; + } + + if (path === '/api/unread/mark-all-read' && req.method === 'POST') { + const authUser = await this.requireAuth(req, res); + if (!authUser) return; + const repo = this.storage?.readCursorRepo; + if (!repo) { this.json(res, 200, { success: true }); return; } + repo.markAllRead(authUser.userId); + this.json(res, 200, { success: true }); + return; + } + // Unified activity feed — merges notifications, task comments, and deliverables if (path === '/api/activity' && req.method === 'GET') { const authUser = await this.requireAuth(req, res); @@ -6881,6 +6952,35 @@ EXPLANATION_END`; return; } + // Settings — Remote Access + if (path === '/api/settings/remote' && req.method === 'GET') { + const status = this.remoteAgent?.getStatus() ?? { enabled: false, connected: false, instanceId: null, remoteUrl: null, signalUrl: null, peerCount: 0 }; + this.json(res, 200, status); + return; + } + + if (path === '/api/settings/remote/enable' && req.method === 'POST') { + if (!this.remoteAgent && this.remoteAgentFactory) { + const agent = await this.remoteAgentFactory(); + if (agent) this.setRemoteAgent(agent); + } + if (!this.remoteAgent) { + this.json(res, 400, { error: 'Remote access not configured. Please sign in to Markus Hub first.' }); + return; + } + this.remoteAgent.start().catch(() => {}); + this.json(res, 200, { ok: true, status: this.remoteAgent.getStatus() }); + return; + } + + if (path === '/api/settings/remote/disable' && req.method === 'POST') { + if (this.remoteAgent) { + await this.remoteAgent.stop(); + } + this.json(res, 200, { ok: true }); + return; + } + // Settings — LLM configuration if (path === '/api/settings/llm' && req.method === 'GET') { if (!this.llmRouter) { @@ -6995,10 +7095,14 @@ EXPLANATION_END`; const { loadConfig: loadCfg } = await import('@markus/shared'); const currentConfig = loadCfg(this.markusConfigPath); const browser = currentConfig.browser ?? {}; + const am = this.orgService.getAgentManager(); this.json(res, 200, { bringToFront: browser.bringToFront ?? false, remoteDebuggingPort: browser.remoteDebuggingPort ?? 0, autoCloseTabs: browser.autoCloseTabs ?? true, + autoClickAllowDialog: browser.autoClickAllowDialog ?? false, + extensionBridgePort: browser.extensionBridgePort ?? 9333, + extensionConnected: am.browserExtensionConnected, }); return; } @@ -7011,6 +7115,7 @@ EXPLANATION_END`; if (typeof body['bringToFront'] === 'boolean') updates.bringToFront = body['bringToFront']; if (typeof body['remoteDebuggingPort'] === 'number') updates.remoteDebuggingPort = body['remoteDebuggingPort']; if (typeof body['autoCloseTabs'] === 'boolean') updates.autoCloseTabs = body['autoCloseTabs']; + if (typeof body['autoClickAllowDialog'] === 'boolean') updates.autoClickAllowDialog = body['autoClickAllowDialog']; try { saveConfig({ browser: updates } as any, this.markusConfigPath); const am = this.orgService.getAgentManager(); @@ -7023,6 +7128,9 @@ EXPLANATION_END`; if (typeof updates.remoteDebuggingPort === 'number') { am.setBrowserRemoteDebuggingPort(updates.remoteDebuggingPort); } + if (typeof updates.autoClickAllowDialog === 'boolean') { + am.setBrowserAutoClickAllowDialog(updates.autoClickAllowDialog); + } } catch (e) { log.warn('Failed to persist browser settings', { error: String(e) }); } @@ -7040,14 +7148,98 @@ EXPLANATION_END`; const { loadConfig: loadCfg } = await import('@markus/shared'); const currentConfig = loadCfg(this.markusConfigPath); const browser = currentConfig.browser ?? {}; + const am2 = this.orgService.getAgentManager(); this.json(res, 200, { bringToFront: browser.bringToFront ?? false, remoteDebuggingPort: browser.remoteDebuggingPort ?? 0, autoCloseTabs: browser.autoCloseTabs ?? true, + autoClickAllowDialog: browser.autoClickAllowDialog ?? false, + extensionBridgePort: browser.extensionBridgePort ?? 9333, + extensionConnected: am2.browserExtensionConnected, }); return; } + // Settings — Chrome Extension: download zip + if (path === '/api/settings/browser/extension.zip' && req.method === 'GET') { + const auth = await this.requireAuth(req, res); + if (!auth) return; + + try { + const { fileURLToPath } = await import('node:url'); + const { dirname: dn, resolve: rslv, join: jn } = await import('node:path'); + const { execSync } = await import('node:child_process'); + const { existsSync: ex, readFileSync, statSync } = await import('node:fs'); + + const thisDir = dn(fileURLToPath(import.meta.url)); + // Search order: dev workspace → binary install → cwd fallback + const zipCandidates = [ + jn(rslv(thisDir, '..', '..', 'chrome-extension'), 'dist', 'markus-browser-extension.zip'), + jn(rslv(thisDir, '..', '..', '..', 'chrome-extension'), 'markus-browser-extension.zip'), + jn(rslv(process.cwd(), 'packages', 'chrome-extension'), 'dist', 'markus-browser-extension.zip'), + jn(rslv(process.cwd(), 'chrome-extension'), 'markus-browser-extension.zip'), + ]; + let zipPath = zipCandidates.find(p => ex(p)); + + // If not found, try building from source + if (!zipPath) { + const extDir = [ + rslv(thisDir, '..', '..', 'chrome-extension'), + rslv(process.cwd(), 'packages', 'chrome-extension'), + ].find(d => ex(jn(d, 'package.json'))); + if (extDir) { + try { execSync('pnpm run pack', { cwd: extDir, timeout: 30000, stdio: 'pipe' }); } catch { /* ignore */ } + const built = jn(extDir, 'dist', 'markus-browser-extension.zip'); + if (ex(built)) zipPath = built; + } + } + if (!zipPath) { this.json(res, 404, { error: 'Extension zip not found.' }); return; } + + const data = readFileSync(zipPath); + res.writeHead(200, { + 'Content-Type': 'application/zip', + 'Content-Disposition': 'attachment; filename="markus-browser-extension.zip"', + 'Content-Length': data.length, + }); + res.end(data); + } catch (e) { + this.json(res, 500, { error: String(e) }); + } + return; + } + + // Settings — Chrome Extension: open chrome://extensions page + if (path === '/api/settings/browser/open-extensions-page' && req.method === 'POST') { + const auth = await this.requireAuth(req, res); + if (!auth) return; + + try { + const { exec: execCb } = await import('node:child_process'); + const platform = process.platform; + if (platform === 'darwin') { + execCb('open -a "Google Chrome" "chrome://extensions"', () => {}); + } else if (platform === 'win32') { + execCb('start chrome "chrome://extensions"', () => {}); + } else { + execCb('xdg-open "chrome://extensions" 2>/dev/null || google-chrome "chrome://extensions"', () => {}); + } + this.json(res, 200, { ok: true }); + } catch (e) { + this.json(res, 500, { error: String(e) }); + } + return; + } + + // Settings — Browser auto-click test + if (path === '/api/settings/browser/test-auto-click' && req.method === 'POST') { + const auth = await this.requireAuth(req, res); + if (!auth) return; + const { testAutoClick } = await import('@markus/core'); + const result = await testAutoClick(); + this.json(res, 200, result); + return; + } + // Settings — Search API keys if (path === '/api/settings/search' && req.method === 'GET') { const { loadConfig: loadCfg } = await import('@markus/shared'); @@ -9643,6 +9835,7 @@ EXPLANATION_END`; exact('/api/settings/agent', 'GET', 'POST'), exact('/api/settings/browser', 'GET', 'POST'), exact('/api/settings/browser/check', 'GET'), + exact('/api/settings/browser/test-auto-click', 'POST'), exact('/api/settings/search', 'GET', 'POST'), exact('/api/settings/env-models', 'GET', 'POST'), exact('/api/settings/detect-ollama', 'GET'), diff --git a/packages/org-manager/src/storage-bridge.ts b/packages/org-manager/src/storage-bridge.ts index 3efea5ea..16c02d57 100644 --- a/packages/org-manager/src/storage-bridge.ts +++ b/packages/org-manager/src/storage-bridge.ts @@ -37,6 +37,7 @@ export interface StorageBridge { groupChatRepo?: any; auditRepo?: any; statusTransitionRepo?: any; + readCursorRepo?: any; } function resolveSqlitePath(url?: string): string { @@ -86,6 +87,7 @@ async function initSqliteStorage(url?: string): Promise { groupChatRepo: new storage.SqliteGroupChatRepo(db), auditRepo: new storage.SqliteAuditRepo(db), statusTransitionRepo: new storage.SqliteStatusTransitionRepo(db), + readCursorRepo: new storage.SqliteReadCursorRepo(db), }; log.info('SQLite storage initialized', { path: dbPath }); return bridge; diff --git a/packages/org-manager/src/ws-server.ts b/packages/org-manager/src/ws-server.ts index abddc47a..90a81716 100644 --- a/packages/org-manager/src/ws-server.ts +++ b/packages/org-manager/src/ws-server.ts @@ -200,6 +200,14 @@ export class WSBroadcaster { } } + broadcastUnreadUpdate(conversationKey: string, messageId: string): void { + this.broadcast({ + type: 'chat:unread_update', + payload: { conversationKey, messageId }, + timestamp: new Date().toISOString(), + }); + } + broadcastExecutionLog(entry: Record): void { const ts = new Date().toISOString(); this.broadcast({ type: 'execution:log', payload: entry, timestamp: ts }); diff --git a/packages/remote/package.json b/packages/remote/package.json new file mode 100644 index 00000000..d721063f --- /dev/null +++ b/packages/remote/package.json @@ -0,0 +1,21 @@ +{ + "name": "@markus/remote", + "version": "0.6.8", + "private": true, + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "scripts": { + "build": "tsc -b", + "dev": "tsc -b --watch", + "clean": "rm -rf dist *.tsbuildinfo" + }, + "dependencies": { + "@markus/shared": "workspace:*", + "node-datachannel": "^0.12.0", + "ws": "^8.19.0" + }, + "devDependencies": { + "@types/ws": "^8.18.1" + } +} diff --git a/packages/remote/src/agent.ts b/packages/remote/src/agent.ts new file mode 100644 index 00000000..78679e30 --- /dev/null +++ b/packages/remote/src/agent.ts @@ -0,0 +1,872 @@ +import { createLogger } from '@markus/shared'; +import { WebSocket } from 'ws'; +import { request as httpRequest, type IncomingMessage } from 'node:http'; +import { request as httpsRequest } from 'node:https'; +import { createHmac } from 'node:crypto'; +import { + PeerConnection, + type DataChannel, + initLogger as initRtcLogger, + type RtcConfig, + type IceServer, + DescriptionType, +} from 'node-datachannel'; + +const log = createLogger('remote'); + +function base64url(buf: Buffer): string { + return buf.toString('base64').replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); +} + +function signJwt(payload: Record, secret: string): string { + const header = base64url(Buffer.from(JSON.stringify({ alg: 'HS256', typ: 'JWT' }))); + const body = base64url(Buffer.from(JSON.stringify(payload))); + const sig = base64url(createHmac('sha256', secret).update(`${header}.${body}`).digest()); + return `${header}.${body}.${sig}`; +} + +const STUN_SERVERS = [ + 'stun:stun.l.google.com:19302', + 'stun:stun.cloudflare.com:3478', +]; + +const RECONNECT_BASE_MS = 2_000; +const RECONNECT_MAX_MS = 60_000; +const HEARTBEAT_INTERVAL_MS = 25_000; +const PEER_PING_INTERVAL_MS = 15_000; +const PEER_PING_TIMEOUT_MS = 10_000; +const RELAY_INACTIVITY_TIMEOUT_MS = 5 * 60_000; + +export interface RemoteAccessConfig { + hubUrl: string; + hubToken: string; + instanceName?: string; + localPort: number; + jwtSecret?: string; +} + +export interface RemotePeerInfo { + peerId: string; + transport: 'p2p' | 'relay' | 'connecting'; + connectedAt: number; + lastActiveAt: number; +} + +export interface RemoteAccessStatus { + enabled: boolean; + connected: boolean; + state: 'idle' | 'registering' | 'connecting' | 'connected' | 'disconnected'; + instanceId: string | null; + remoteUrl: string | null; + signalUrl: string | null; + peerCount: number; + peers: RemotePeerInfo[]; +} + +interface TurnServer { + urls: string; + username: string; + credential: string; +} + +interface RegistrationResult { + instanceId: string; + signalingToken: string; + signalUrl: string; + remoteUrl: string; + turnServers?: TurnServer[] | null; +} + +interface PeerSession { + pc: PeerConnection | null; + dc: DataChannel | null; + pendingChunks: Map; + markusToken: string | null; + connectedAt: number; + lastActiveAt: number; + pingTimer: ReturnType | null; + lastPong: number; +} + +export class RemoteAccessAgent { + private config: RemoteAccessConfig; + private ws: WebSocket | null = null; + private registration: RegistrationResult | null = null; + private peers = new Map(); + private heartbeatTimer: ReturnType | null = null; + private reconnectTimer: ReturnType | null = null; + private reconnectAttempts = 0; + private destroyed = false; + private localOwnerUserId: string | null = null; + + private statusListeners = new Set<(status: RemoteAccessStatus) => void>(); + + constructor(config: RemoteAccessConfig) { + this.config = config; + initRtcLogger('Warning'); + } + + // ── Public API ──────────────────────────────────────────────────────────── + + async start(): Promise { + this.destroyed = false; + log.info('Starting remote access agent...'); + + await this.discoverLocalOwner(); + + try { + this.registration = await this.registerInstance(); + log.info('Registered with Hub', { + instanceId: this.registration.instanceId, + remoteUrl: this.registration.remoteUrl, + }); + this.connectSignaling(); + } catch (err) { + log.error('Failed to register with Hub', { error: String(err) }); + this.scheduleReconnect(); + } + } + + private async discoverLocalOwner(): Promise { + for (let attempt = 0; attempt < 3; attempt++) { + try { + const resp = await new Promise((resolve, reject) => { + const req = httpRequest( + `http://127.0.0.1:${this.config.localPort}/api/users`, + { method: 'GET', headers: { host: `127.0.0.1:${this.config.localPort}` } }, + (res) => { + const chunks: Buffer[] = []; + res.on('data', (c: Buffer) => chunks.push(c)); + res.on('end', () => resolve(Buffer.concat(chunks).toString())); + } + ); + req.on('error', reject); + req.end(); + }); + + const data = JSON.parse(resp); + const users = data.users as Array<{ id: string; role: string }> | undefined; + if (users?.length) { + const owner = users.find(u => u.role === 'owner') ?? users[0]; + this.localOwnerUserId = owner!.id; + log.info('Discovered local owner', { userId: this.localOwnerUserId }); + } + return; + } catch (err) { + if (attempt < 2) { + await new Promise(r => setTimeout(r, 1000 * (attempt + 1))); + } else { + log.warn('Failed to discover local owner, using synthetic user', { error: String(err) }); + } + } + } + } + + async stop(): Promise { + this.destroyed = true; + if (this.heartbeatTimer) clearInterval(this.heartbeatTimer); + if (this.reconnectTimer) clearTimeout(this.reconnectTimer); + + for (const [peerId, session] of this.peers) { + try { session.dc?.close(); } catch { /* ignore */ } + try { session.pc?.close(); } catch { /* ignore */ } + this.peers.delete(peerId); + } + + if (this.ws) { + this.ws.close(1000, 'shutdown'); + this.ws = null; + } + + if (this.registration) { + await this.unregisterInstance().catch(() => {}); + this.registration = null; + } + + this.emitStatus(); + log.info('Remote access agent stopped'); + } + + getStatus(): RemoteAccessStatus { + const wsOpen = this.ws?.readyState === WebSocket.OPEN; + let state: RemoteAccessStatus['state'] = 'idle'; + if (!this.destroyed) { + if (wsOpen) state = 'connected'; + else if (this.registration) state = 'connecting'; + else if (this.reconnectTimer) state = 'connecting'; + else state = 'registering'; + } + + const peers: RemotePeerInfo[] = []; + for (const [peerId, session] of this.peers) { + let transport: 'p2p' | 'relay' | 'connecting' = 'connecting'; + if (session.dc && session.dc.isOpen()) { + transport = 'p2p'; + } else if (wsOpen) { + transport = 'relay'; + } + peers.push({ + peerId, + transport, + connectedAt: session.connectedAt, + lastActiveAt: session.lastActiveAt, + }); + } + + return { + enabled: !this.destroyed, + connected: wsOpen, + state, + instanceId: this.registration?.instanceId ?? null, + remoteUrl: this.registration?.remoteUrl ?? null, + signalUrl: this.registration?.signalUrl ?? null, + peerCount: this.peers.size, + peers, + }; + } + + onStatus(listener: (status: RemoteAccessStatus) => void): () => void { + this.statusListeners.add(listener); + return () => this.statusListeners.delete(listener); + } + + // ── Hub Registration ────────────────────────────────────────────────────── + + private async registerInstance(): Promise { + const resp = await this.hubFetch('POST', '/api/remote/instances', { + name: this.config.instanceName ?? 'My Markus', + }); + return resp as RegistrationResult; + } + + private async unregisterInstance(): Promise { + if (!this.registration) return; + await this.hubFetch('DELETE', '/api/remote/instances', { + instanceId: this.registration.instanceId, + }); + } + + private hubFetch(method: string, path: string, body?: unknown, _redirects = 0): Promise { + return new Promise((resolve, reject) => { + const url = new URL(path, this.config.hubUrl); + const data = body ? JSON.stringify(body) : undefined; + const transport = url.protocol === 'https:' ? httpsRequest : httpRequest; + + const req = transport( + url, + { + method, + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.config.hubToken}`, + ...(data ? { 'Content-Length': Buffer.byteLength(data).toString() } : {}), + }, + }, + (res: IncomingMessage) => { + if (res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { + if (_redirects >= 5) { reject(new Error('Too many redirects')); return; } + const redirectUrl = new URL(res.headers.location, url); + this.config.hubUrl = redirectUrl.origin; + resolve(this.hubFetch(method, redirectUrl.pathname + redirectUrl.search, body, _redirects + 1)); + return; + } + let raw = ''; + res.on('data', (c: Buffer) => (raw += c.toString())); + res.on('end', () => { + try { + const json = JSON.parse(raw); + if (res.statusCode && res.statusCode >= 400) { + reject(new Error(json.error ?? `HTTP ${res.statusCode}`)); + } else { + resolve(json); + } + } catch { + reject(new Error(`Invalid JSON response: ${raw.slice(0, 200)}`)); + } + }); + } + ); + + req.on('error', reject); + if (data) req.write(data); + req.end(); + }); + } + + // ── Signaling WebSocket ─────────────────────────────────────────────────── + + private connectSignaling(): void { + if (this.destroyed || !this.registration) return; + + const { signalUrl, signalingToken } = this.registration; + const wsUrl = `${signalUrl}?token=${encodeURIComponent(signalingToken)}`; + + log.info('Connecting to signal server...', { signalUrl }); + + const ws = new WebSocket(wsUrl); + this.ws = ws; + + ws.on('open', () => { + log.info('Signal server connected'); + this.reconnectAttempts = 0; + this.startHeartbeat(); + this.emitStatus(); + + this.send({ type: 'register', instanceId: this.registration!.instanceId }); + }); + + ws.on('message', (data: Buffer) => { + try { + const msg = JSON.parse(data.toString()); + this.handleSignalingMessage(msg); + } catch (err) { + log.warn('Invalid signaling message', { error: String(err) }); + } + }); + + ws.on('close', (code: number) => { + log.warn('Signal server disconnected', { code }); + this.stopHeartbeat(); + this.ws = null; + this.emitStatus(); + if (!this.destroyed) this.scheduleReconnect(); + }); + + ws.on('error', (err: Error) => { + log.error('Signal server error', { error: err.message }); + }); + } + + private handleSignalingMessage(msg: Record): void { + const type = msg['type'] as string; + const peerId = (msg['peerId'] ?? msg['from']) as string | undefined; + + switch (type) { + case 'ping': + this.send({ type: 'pong' }); + break; + case 'registered': + log.info('Registered with signal server', { instanceId: msg['instanceId'] }); + break; + case 'peer_request': + if (peerId) this.handlePeerRequest(peerId); + break; + case 'offer': + if (peerId && msg['sdp']) { + this.handleOffer(peerId, msg['sdp'] as string); + } + break; + case 'ice': + if (peerId && msg['candidate']) { + this.handleIce(peerId, msg['candidate'] as string, msg['mid'] as string | undefined); + } + break; + case 'peer_disconnected': + if (peerId) this.cleanupPeer(peerId); + break; + case 'relay_activated': + if (peerId) log.info('Peer activated relay mode', { peerId }); + break; + case 'relay_frame': + if (peerId && msg['data']) { + this.handleRelayFrame(peerId, msg['data'] as string); + } + break; + default: + log.debug('Unknown signaling message type', { type }); + } + } + + // ── WebRTC Peer Connections ─────────────────────────────────────────────── + + private handlePeerRequest(peerId: string): void { + log.info('Peer connection requested', { peerId }); + this.createPeerConnection(peerId); + } + + private handleOffer(peerId: string, sdp: string): void { + let session = this.peers.get(peerId); + if (!session) { + log.info('Received offer, creating new peer connection', { peerId }); + session = this.createPeerConnection(peerId); + } else if (!session.pc) { + log.info('Received offer for relay-only peer, upgrading to P2P', { peerId }); + const newSession = this.createPeerConnection(peerId); + newSession.markusToken = session.markusToken; + newSession.connectedAt = session.connectedAt; + newSession.lastActiveAt = session.lastActiveAt; + if (session.pingTimer) clearInterval(session.pingTimer); + session = newSession; + } else { + log.info('Received offer for existing peer (ICE restart)', { peerId }); + } + + session.pc!.setRemoteDescription(sdp, DescriptionType.Offer); + } + + private handleIce(peerId: string, candidate: string, mid?: string): void { + const session = this.peers.get(peerId); + if (!session?.pc) { + log.warn('Received ICE candidate but no PC', { peerId, hasSession: !!session }); + return; + } + session.pc.addRemoteCandidate(candidate, mid ?? '0'); + } + + private createPeerConnection(peerId: string): PeerSession { + if (this.peers.has(peerId)) { + this.cleanupPeer(peerId); + } + + const iceServers: (string | IceServer)[] = [...STUN_SERVERS]; + if (this.registration?.turnServers) { + for (const t of this.registration.turnServers) { + const parsed = t.urls.match(/^(turns?):([^:?]+):(\d+)/); + if (parsed) { + const isTcp = t.urls.includes('transport=tcp'); + const isTls = parsed[1] === 'turns'; + let relayType = 'TurnUdp'; + if (isTls) relayType = 'TurnTls'; + else if (isTcp) relayType = 'TurnTcp'; + iceServers.push({ + hostname: parsed[2]!, + port: parseInt(parsed[3]!, 10), + username: t.username, + password: t.credential, + relayType, + } as IceServer); + } + } + } + const pc = new PeerConnection(`markus-${peerId}`, { + iceServers, + } satisfies RtcConfig); + + const now = Date.now(); + const session: PeerSession = { pc, dc: null, pendingChunks: new Map(), markusToken: null, connectedAt: now, lastActiveAt: now, pingTimer: null, lastPong: now }; + this.peers.set(peerId, session); + + pc.onStateChange((state: string) => { + log.info('Peer RTC state', { peerId, state }); + if (state === 'failed' || state === 'closed') { + this.handlePcFailed(peerId); + } + this.emitStatus(); + }); + + pc.onGatheringStateChange((state: string) => { + log.info('ICE gathering', { peerId, state }); + }); + + pc.onLocalDescription((sdp: string, type: DescriptionType) => { + log.info('Sending local description', { peerId, type: type as string }); + this.send({ type: type as string, peerId, sdp }); + }); + + pc.onLocalCandidate((candidate: string, mid: string) => { + log.info('Sending ICE candidate', { peerId, candidate: candidate.slice(0, 60) }); + this.send({ type: 'ice', peerId, candidate, mid }); + }); + + pc.onDataChannel((dc: DataChannel) => { + log.info('DataChannel opened', { peerId, label: dc.getLabel() }); + session.dc = dc; + session.lastPong = Date.now(); + this.emitStatus(); + + dc.onMessage((msg: string | Buffer) => { + const data = typeof msg === 'string' ? msg : msg.toString('utf-8'); + session.lastActiveAt = Date.now(); + this.handleDataChannelMessage(peerId, data); + }); + + dc.onClosed(() => { + log.info('DataChannel closed, keeping session for relay', { peerId }); + session.dc = null; + this.emitStatus(); + }); + }); + + return session; + } + + private handlePcFailed(peerId: string): void { + const session = this.peers.get(peerId); + if (!session) return; + + log.info('WebRTC failed, keeping session alive for relay', { peerId }); + try { session.dc?.close(); } catch { /* ignore */ } + session.dc = null; + try { session.pc?.close(); } catch { /* ignore */ } + session.pc = null; + } + + private cleanupPeer(peerId: string): void { + const session = this.peers.get(peerId); + if (!session) return; + + if (session.pingTimer) clearInterval(session.pingTimer); + try { session.dc?.close(); } catch { /* ignore */ } + try { session.pc?.close(); } catch { /* ignore */ } + for (const [key, ws] of this.wsConnections) { + if (key.startsWith(`${peerId}:`)) { + ws.close(); + this.wsConnections.delete(key); + } + } + this.peers.delete(peerId); + this.emitStatus(); + log.info('Peer cleaned up', { peerId }); + } + + private startPeerPing(peerId: string, session: PeerSession): void { + if (session.pingTimer) clearInterval(session.pingTimer); + session.lastPong = Date.now(); + + // Send first ping immediately so we get a pong before the first timeout check + this.sendRaw(peerId, JSON.stringify({ type: '__ping' })); + + session.pingTimer = setInterval(() => { + const now = Date.now(); + + // Check inactivity — clean up if no messages for RELAY_INACTIVITY_TIMEOUT_MS + if (now - session.lastActiveAt > RELAY_INACTIVITY_TIMEOUT_MS) { + log.info('Peer inactive for too long, cleaning up', { peerId }); + this.cleanupPeer(peerId); + return; + } + + // Check pong timeout — only if we've sent at least one ping and waited long enough + const elapsed = now - session.lastPong; + if (elapsed > PEER_PING_INTERVAL_MS + PEER_PING_TIMEOUT_MS) { + log.warn('Peer ping timeout, unresponsive', { peerId, elapsed }); + this.cleanupPeer(peerId); + return; + } + + // Send ping via whatever transport is available (DC or relay) + this.sendRaw(peerId, JSON.stringify({ type: '__ping' })); + }, PEER_PING_INTERVAL_MS); + } + + // ── DataChannel Message Handling (HTTP/WS proxy) ───────────────────────── + + private handleDataChannelMessage(peerId: string, raw: string): void { + const session = this.peers.get(peerId); + if (session) session.lastActiveAt = Date.now(); + + try { + const msg = JSON.parse(raw); + const type = msg.type as string; + + switch (type) { + case '__pong': { + const s = this.peers.get(peerId); + if (s) s.lastPong = Date.now(); + return; + } + case 'http': + this.proxyHttpRequest(peerId, msg); + break; + case 'ws_open': + this.proxyWsOpen(peerId, msg); + break; + case 'ws_message': + this.proxyWsMessage(peerId, msg); + break; + case 'ws_close': + this.proxyWsClose(peerId, msg); + break; + case 'auth': + this.handleAuthHandshake(peerId, msg); + break; + default: + this.sendToPeer(peerId, { type: 'error', error: `Unknown message type: ${type}` }); + } + } catch (err) { + log.warn('Invalid DataChannel message', { peerId, error: String(err) }); + } + } + + private handleRelayFrame(peerId: string, data: string): void { + if (!this.peers.has(peerId)) { + log.info('Relay frame from unknown peer, creating relay-only session', { peerId }); + const now = Date.now(); + this.peers.set(peerId, { + pc: null, + dc: null, + pendingChunks: new Map(), + markusToken: null, + connectedAt: now, + lastActiveAt: now, + pingTimer: null, + lastPong: now, + }); + this.emitStatus(); + } + this.handleDataChannelMessage(peerId, data); + } + + private generateMarkusToken(): string { + const secret = this.config.jwtSecret ?? process.env['JWT_SECRET'] ?? 'markus-dev-secret-change-in-prod'; + const exp = Math.floor(Date.now() / 1000) + 24 * 3600; + const userId = this.localOwnerUserId ?? 'remote_owner'; + return signJwt({ userId, orgId: 'default', role: 'owner', exp }, secret); + } + + private handleAuthHandshake(peerId: string, _msg: Record): void { + const session = this.peers.get(peerId); + if (session && !session.markusToken) { + session.markusToken = this.generateMarkusToken(); + } + if (session && !session.pingTimer) { + this.startPeerPing(peerId, session); + } + this.sendToPeer(peerId, { + type: 'auth_ok', + instanceName: this.config.instanceName ?? 'My Markus', + token: session?.markusToken ?? null, + }); + } + + private proxyHttpRequest(peerId: string, msg: Record): void { + const reqId = msg.id as string; + const method = (msg.method as string) ?? 'GET'; + const path = (msg.path as string) ?? '/'; + const headers = (msg.headers as Record) ?? {}; + const body = msg.body as string | undefined; + + const session = this.peers.get(peerId); + if (session && !session.markusToken) { + session.markusToken = this.generateMarkusToken(); + } + const tokenCookie = session?.markusToken ? `markus_token=${session.markusToken}` : ''; + const existingCookie = headers['cookie'] ?? headers['Cookie'] ?? ''; + const cookie = existingCookie ? `${existingCookie}; ${tokenCookie}` : tokenCookie; + + const url = new URL(path, `http://127.0.0.1:${this.config.localPort}`); + + const req = httpRequest( + url, + { + method, + headers: { ...headers, host: `127.0.0.1:${this.config.localPort}`, cookie }, + }, + (res: IncomingMessage) => { + const contentType = res.headers['content-type'] ?? ''; + const isStreaming = contentType.includes('text/event-stream') || + contentType.includes('application/x-ndjson') || + res.headers['transfer-encoding'] === 'chunked' && contentType.includes('stream'); + + if (isStreaming) { + this.sendToPeer(peerId, { + type: 'http_response_start', + id: reqId, + status: res.statusCode ?? 200, + headers: res.headers, + }); + + res.on('data', (c: Buffer) => { + this.sendToPeer(peerId, { + type: 'http_response_chunk', + id: reqId, + data: c.toString('base64'), + }); + }); + + res.on('end', () => { + this.sendToPeer(peerId, { + type: 'http_response_end', + id: reqId, + }); + }); + + res.on('error', () => { + this.sendToPeer(peerId, { + type: 'http_response_end', + id: reqId, + }); + }); + } else { + const chunks: Buffer[] = []; + res.on('data', (c: Buffer) => chunks.push(c)); + res.on('end', () => { + const bodyStr = Buffer.concat(chunks).toString('base64'); + this.sendToPeer(peerId, { + type: 'http_response', + id: reqId, + status: res.statusCode ?? 200, + headers: res.headers, + body: bodyStr, + }); + }); + } + } + ); + + req.on('error', (err: Error) => { + this.sendToPeer(peerId, { + type: 'http_response', + id: reqId, + status: 502, + headers: {}, + body: Buffer.from(JSON.stringify({ error: err.message })).toString('base64'), + }); + }); + + if (body) req.write(Buffer.from(body, 'base64')); + req.end(); + } + + private wsConnections = new Map(); + + private proxyWsOpen(peerId: string, msg: Record): void { + const wsId = msg.wsId as string; + const path = (msg.path as string) ?? '/ws'; + const wsUrl = `ws://127.0.0.1:${this.config.localPort}${path}`; + + const session = this.peers.get(peerId); + if (session && !session.markusToken) { + session.markusToken = this.generateMarkusToken(); + } + const headers: Record = {}; + if (session?.markusToken) { + headers['cookie'] = `markus_token=${session.markusToken}`; + } + + const ws = new WebSocket(wsUrl, { headers }); + const key = `${peerId}:${wsId}`; + + ws.on('open', () => { + this.wsConnections.set(key, ws); + this.sendToPeer(peerId, { type: 'ws_opened', wsId }); + }); + + ws.on('message', (data: Buffer) => { + this.sendToPeer(peerId, { + type: 'ws_frame', + wsId, + data: data.toString('utf-8'), + }); + }); + + ws.on('close', (code: number) => { + this.wsConnections.delete(key); + this.sendToPeer(peerId, { type: 'ws_closed', wsId, code }); + }); + + ws.on('error', (err: Error) => { + this.wsConnections.delete(key); + this.sendToPeer(peerId, { type: 'ws_error', wsId, error: err.message }); + }); + } + + private proxyWsMessage(peerId: string, msg: Record): void { + const wsId = msg.wsId as string; + const key = `${peerId}:${wsId}`; + const ws = this.wsConnections.get(key); + if (ws?.readyState === WebSocket.OPEN) { + ws.send(msg.data as string); + } + } + + private proxyWsClose(peerId: string, msg: Record): void { + const wsId = msg.wsId as string; + const key = `${peerId}:${wsId}`; + const ws = this.wsConnections.get(key); + if (ws) { + ws.close(); + this.wsConnections.delete(key); + } + } + + // ── Helpers ─────────────────────────────────────────────────────────────── + + private static readonly CHUNK_SIZE = 48 * 1024; // 48KB per chunk (safe for DC + relay) + + private sendToPeer(peerId: string, msg: unknown): void { + const data = JSON.stringify(msg); + + if (data.length > RemoteAccessAgent.CHUNK_SIZE) { + this.sendChunked(peerId, data); + return; + } + + this.sendRaw(peerId, data); + } + + private sendChunked(peerId: string, data: string): void { + const chunkId = `c${Date.now().toString(36)}${Math.random().toString(36).slice(2, 6)}`; + const total = Math.ceil(data.length / RemoteAccessAgent.CHUNK_SIZE); + + for (let i = 0; i < total; i++) { + const chunk = data.slice(i * RemoteAccessAgent.CHUNK_SIZE, (i + 1) * RemoteAccessAgent.CHUNK_SIZE); + this.sendRaw(peerId, JSON.stringify({ + type: '__chunk', + chunkId, + index: i, + total, + data: chunk, + })); + } + } + + private sendRaw(peerId: string, data: string): void { + // Prefer P2P DataChannel — direct, low latency + const session = this.peers.get(peerId); + if (session?.dc && session.dc.isOpen()) { + try { + session.dc.sendMessage(data); + return; + } catch (err) { + log.warn('DataChannel send failed, falling back to relay', { peerId, error: String(err) }); + } + } + + // Fallback to relay via signaling WS + if (this.ws?.readyState === WebSocket.OPEN) { + this.send({ type: 'relay_frame', peerId, data }); + return; + } + + log.warn('No transport available for peer', { peerId }); + } + + private send(msg: unknown): void { + if (this.ws?.readyState === WebSocket.OPEN) { + this.ws.send(JSON.stringify(msg)); + } + } + + private startHeartbeat(): void { + this.stopHeartbeat(); + this.heartbeatTimer = setInterval(() => { + this.send({ type: 'pong' }); + }, HEARTBEAT_INTERVAL_MS); + } + + private stopHeartbeat(): void { + if (this.heartbeatTimer) { + clearInterval(this.heartbeatTimer); + this.heartbeatTimer = null; + } + } + + private scheduleReconnect(): void { + if (this.destroyed) return; + const delay = Math.min( + RECONNECT_BASE_MS * Math.pow(2, this.reconnectAttempts), + RECONNECT_MAX_MS + ); + this.reconnectAttempts++; + log.info(`Reconnecting in ${delay}ms (attempt ${this.reconnectAttempts})...`); + this.reconnectTimer = setTimeout(() => this.start(), delay); + } + + private emitStatus(): void { + const status = this.getStatus(); + for (const listener of this.statusListeners) { + try { listener(status); } catch { /* ignore */ } + } + } +} diff --git a/packages/remote/src/index.ts b/packages/remote/src/index.ts new file mode 100644 index 00000000..1e8ffb93 --- /dev/null +++ b/packages/remote/src/index.ts @@ -0,0 +1 @@ +export { RemoteAccessAgent, type RemoteAccessConfig, type RemoteAccessStatus } from './agent.js'; diff --git a/packages/remote/tsconfig.json b/packages/remote/tsconfig.json new file mode 100644 index 00000000..d3ddf396 --- /dev/null +++ b/packages/remote/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "composite": true, + "declaration": true, + "declarationMap": true + }, + "include": ["src"], + "references": [ + { "path": "../shared" } + ] +} diff --git a/packages/shared/src/utils/config.ts b/packages/shared/src/utils/config.ts index 58e6c8c5..112f9391 100644 --- a/packages/shared/src/utils/config.ts +++ b/packages/shared/src/utils/config.ts @@ -69,6 +69,10 @@ export interface MarkusConfig { remoteDebuggingPort?: number; /** Automatically close agent-owned tabs when the task completes (default: true) */ autoCloseTabs?: boolean; + /** Auto-click Chrome's "Allow debugging" dialog via OS accessibility APIs (macOS/Windows) */ + autoClickAllowDialog?: boolean; + /** WebSocket port for Chrome extension bridge (default: 9333) */ + extensionBridgePort?: number; }; integrations?: { feishu?: { appId?: string; appSecret?: string }; @@ -89,6 +93,12 @@ export interface MarkusConfig { // Future cloud providers: // s3?: { bucket: string; region: string; endpoint?: string; accessKeyId?: string; secretAccessKey?: string }; }; + remote?: { + enabled?: boolean; + autoConnect?: boolean; + hubUrl?: string; + instanceName?: string; + }; } const DEFAULT_CONFIG: MarkusConfig = { diff --git a/packages/storage/src/index.ts b/packages/storage/src/index.ts index efd285b9..aef5f589 100644 --- a/packages/storage/src/index.ts +++ b/packages/storage/src/index.ts @@ -44,6 +44,7 @@ export { SqliteGroupChatRepo, SqliteAuditRepo, SqliteStatusTransitionRepo, + SqliteReadCursorRepo, migrateToExecutionStreamLogs, type SqliteExternalAgentRegistration, type ActivityRecord, @@ -54,4 +55,5 @@ export { type NotificationRow, type ApprovalRow, type StatusTransitionRow, + type ReadCursorRow, } from './sqlite-storage.js'; diff --git a/packages/storage/src/sqlite-storage.ts b/packages/storage/src/sqlite-storage.ts index e95b989d..64afc2c4 100644 --- a/packages/storage/src/sqlite-storage.ts +++ b/packages/storage/src/sqlite-storage.ts @@ -585,6 +585,15 @@ CREATE TABLE IF NOT EXISTS status_transitions ( created_at TEXT NOT NULL DEFAULT (datetime('now')) ); CREATE INDEX IF NOT EXISTS idx_st_entity ON status_transitions(entity_type, entity_id, created_at); + +CREATE TABLE IF NOT EXISTS user_read_cursors ( + user_id TEXT NOT NULL, + conversation_key TEXT NOT NULL, + last_read_at TEXT NOT NULL, + last_read_id TEXT, + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (user_id, conversation_key) +); `; // ─── Open / close ──────────────────────────────────────────────────────────── @@ -4320,6 +4329,90 @@ export class SqliteStatusTransitionRepo { } } +// ─── Read Cursors (unread tracking) ────────────────────────────────────────── + +export interface ReadCursorRow { + userId: string; + conversationKey: string; + lastReadAt: string; + lastReadId: string | null; + updatedAt: string; +} + +export class SqliteReadCursorRepo { + constructor(private db: DatabaseSync) {} + + setReadCursor(userId: string, conversationKey: string, lastReadAt: string, lastReadId?: string): void { + this.db.prepare( + `INSERT INTO user_read_cursors (user_id, conversation_key, last_read_at, last_read_id, updated_at) + VALUES (?, ?, ?, ?, datetime('now')) + ON CONFLICT(user_id, conversation_key) DO UPDATE SET + last_read_at = excluded.last_read_at, + last_read_id = COALESCE(excluded.last_read_id, last_read_id), + updated_at = datetime('now')` + ).run(userId, conversationKey, lastReadAt, lastReadId ?? null); + } + + getReadCursors(userId: string): ReadCursorRow[] { + return this.db.prepare( + `SELECT user_id AS userId, conversation_key AS conversationKey, + last_read_at AS lastReadAt, last_read_id AS lastReadId, + updated_at AS updatedAt + FROM user_read_cursors WHERE user_id = ?` + ).all(userId) as unknown as ReadCursorRow[]; + } + + getUnreadCounts(userId: string): Record { + const cursors = this.getReadCursors(userId); + const result: Record = {}; + + for (const cursor of cursors) { + const key = cursor.conversationKey; + if (key.startsWith('session:')) { + const sessionId = key.slice('session:'.length); + const row = this.db.prepare( + `SELECT COUNT(*) as cnt FROM chat_messages + WHERE session_id = ? AND created_at > ?` + ).get(sessionId, cursor.lastReadAt) as { cnt: number } | undefined; + if (row && row.cnt > 0) result[key] = row.cnt; + } else if (key.startsWith('channel:')) { + const channel = key.slice('channel:'.length); + const row = this.db.prepare( + `SELECT COUNT(*) as cnt FROM channel_messages + WHERE channel = ? AND created_at > ?` + ).get(channel, cursor.lastReadAt) as { cnt: number } | undefined; + if (row && row.cnt > 0) result[key] = row.cnt; + } + } + + return result; + } + + markAllRead(userId: string): void { + const ts = now(); + // Update all existing cursors + this.db.prepare( + `UPDATE user_read_cursors SET last_read_at = ?, updated_at = datetime('now') WHERE user_id = ?` + ).run(ts, userId); + + // Create cursors for sessions that don't have one + this.db.exec(` + INSERT OR IGNORE INTO user_read_cursors (user_id, conversation_key, last_read_at, updated_at) + SELECT '${userId}', 'session:' || s.id, '${ts}', datetime('now') + FROM chat_sessions s WHERE s.user_id = '${userId}' + `); + + // Create cursors for channels user is a member of + this.db.exec(` + INSERT OR IGNORE INTO user_read_cursors (user_id, conversation_key, last_read_at, updated_at) + SELECT '${userId}', 'channel:' || gc.channel_key, '${ts}', datetime('now') + FROM group_chats gc + JOIN group_chat_members gcm ON gcm.group_chat_id = gc.id + WHERE gcm.member_id = '${userId}' + `); + } +} + // ─── Auto-migration: task_logs + agent_activity_logs -> execution_stream_logs ─ export function migrateToExecutionStreamLogs(db: DatabaseSync): void { diff --git a/packages/web-ui/package.json b/packages/web-ui/package.json index b3474c7a..730df4fa 100644 --- a/packages/web-ui/package.json +++ b/packages/web-ui/package.json @@ -13,10 +13,12 @@ "@dagrejs/dagre": "^2.0.4", "@tailwindcss/typography": "^0.5.19", "@tanstack/react-virtual": "^3.13.24", + "@types/qrcode": "^1.5.6", "@xyflow/react": "^12.10.1", "i18next": "^26.0.6", "i18next-browser-languagedetector": "^8.2.1", "katex": "^0.16.45", + "qrcode": "^1.5.4", "react": "^19.2.4", "react-dom": "^19.2.4", "react-i18next": "^17.0.4", diff --git a/packages/web-ui/src/App.tsx b/packages/web-ui/src/App.tsx index b05cffdd..36f8371a 100644 --- a/packages/web-ui/src/App.tsx +++ b/packages/web-ui/src/App.tsx @@ -9,10 +9,11 @@ import { WorkPage } from './pages/Work.tsx'; import { DeliverablesPage } from './pages/Deliverables.tsx'; import { ReportsPage } from './pages/Reports.tsx'; import { NotificationsPage } from './pages/Notifications.tsx'; +import { SearchPage } from './pages/Search.tsx'; import { Sidebar } from './components/Sidebar.tsx'; import { BottomNav } from './components/BottomNav.tsx'; import { MobileBuilderTabs } from './components/MobileBuilderTabs.tsx'; -import { MobileSettingsTabs } from './components/MobileSettingsTabs.tsx'; +import { MobileDrawer } from './components/MobileDrawer.tsx'; import { Onboarding } from './components/Onboarding.tsx'; import { Login, InviteSetup, InitialSetup } from './pages/Login.tsx'; import { ChangePassword } from './pages/ChangePassword.tsx'; @@ -23,6 +24,7 @@ import { useTheme } from './hooks/useTheme.ts'; import { useIsMobile } from './hooks/useIsMobile.ts'; import { prefetch, PREFETCH_KEYS } from './prefetchCache.ts'; import { useTranslation } from 'react-i18next'; +import { SearchModal } from './components/SearchModal.tsx'; const PageSlot = memo(function PageSlot({ id, activePage, children, @@ -75,6 +77,27 @@ export function App() { return stored ? stored : null; }); + const [showSearchModal, setShowSearchModal] = useState(false); + + // Global search shortcut: Cmd+P (Mac) / Ctrl+P (Win/Linux) + useEffect(() => { + if (isMobile) return; + const isMac = navigator.platform.toUpperCase().includes('MAC'); + const onKey = (e: KeyboardEvent) => { + if (isMac && e.metaKey && !e.ctrlKey && e.key === 'p') { + e.preventDefault(); + setShowSearchModal(prev => !prev); + } else if (!isMac && e.ctrlKey && !e.metaKey && e.key === 'p') { + e.preventDefault(); + setShowSearchModal(prev => !prev); + } + }; + const onOpen = () => setShowSearchModal(true); + document.addEventListener('keydown', onKey); + window.addEventListener('markus:open-search', onOpen); + return () => { document.removeEventListener('keydown', onKey); window.removeEventListener('markus:open-search', onOpen); }; + }, [isMobile]); + const navigate = useCallback((p: PageId) => { let normalized = resolvePageId(p); if (isMobile) { @@ -183,10 +206,12 @@ export function App() { [PAGE.HOME]: , [PAGE.TEAM]: , [PAGE.BUILDER]: , - [PAGE.SETTINGS]: setAuthUser(null)} onUserUpdated={(u) => setAuthUser(u)} />, + [PAGE.SETTINGS]: setAuthUser(null)} onUserUpdated={(u) => setAuthUser(u)} />, [PAGE.WORK]: , [PAGE.DELIVERABLES]: , [PAGE.NOTIFICATIONS]: , + [PAGE.REPORTS]: , + [PAGE.SEARCH]: , }; } return { @@ -272,8 +297,8 @@ export function App() { return (
- {/* Desktop sidebar */} - {!isMobile && ( + {/* Desktop sidebar (hidden on Settings page) */} + {!isMobile && page !== PAGE.SETTINGS && ( <>
)} -
+
{llmConfigured === false && !llmBannerDismissed && page !== PAGE.SETTINGS && (
No LLM provider configured — agents cannot process requests. @@ -329,10 +354,20 @@ export function App() {
- {/* Mobile bottom nav */} - {isMobile && ( + {/* Mobile bottom nav (hidden on Settings/Search pages) */} + {isMobile && page !== PAGE.SETTINGS && page !== PAGE.SEARCH && ( )} + + {/* Mobile drawer menu */} + {isMobile && ( + + )} + + {/* Global search modal (desktop) */} + {!isMobile && showSearchModal && ( + setShowSearchModal(false)} currentPage={page} /> + )}
); } diff --git a/packages/web-ui/src/api.ts b/packages/web-ui/src/api.ts index b4320ca3..25807b64 100644 --- a/packages/web-ui/src/api.ts +++ b/packages/web-ui/src/api.ts @@ -326,6 +326,24 @@ async function request(path: string, opts?: RequestInit): Promise { return res.json() as Promise; } +export interface RemotePeerInfo { + peerId: string; + transport: 'p2p' | 'relay' | 'connecting'; + connectedAt: number; + lastActiveAt: number; +} + +export interface RemoteStatus { + enabled: boolean; + connected: boolean; + state: 'idle' | 'registering' | 'connecting' | 'connected' | 'disconnected'; + instanceId: string | null; + remoteUrl: string | null; + signalUrl: string | null; + peerCount: number; + peers: RemotePeerInfo[]; +} + export type AgentActivityType = 'task' | 'heartbeat' | 'chat' | 'a2a' | 'internal' | 'respond_in_session'; export interface AgentActivityInfo { @@ -1166,12 +1184,40 @@ export const api = { getAgent: () => request<{ maxToolIterations: number; cognitive: { enabled: boolean; maxDepth?: number; appraisalModel?: string; timeoutMs?: number } }>('/settings/agent'), updateAgent: (settings: { maxToolIterations?: number; cognitive?: { enabled?: boolean; maxDepth?: number; appraisalModel?: string; timeoutMs?: number } }) => request<{ maxToolIterations: number; cognitive: { enabled: boolean; maxDepth?: number; appraisalModel?: string; timeoutMs?: number } }>('/settings/agent', { method: 'POST', body: JSON.stringify(settings) }), - getBrowser: () => request<{ bringToFront: boolean; remoteDebuggingPort: number; autoCloseTabs: boolean }>('/settings/browser'), - updateBrowser: (settings: { bringToFront?: boolean; remoteDebuggingPort?: number; autoCloseTabs?: boolean }) => - request<{ bringToFront: boolean; remoteDebuggingPort: number; autoCloseTabs: boolean }>('/settings/browser', { method: 'POST', body: JSON.stringify(settings) }), + getBrowser: () => request<{ bringToFront: boolean; remoteDebuggingPort: number; autoCloseTabs: boolean; autoClickAllowDialog: boolean; extensionBridgePort: number; extensionConnected: boolean }>('/settings/browser'), + updateBrowser: (settings: { bringToFront?: boolean; remoteDebuggingPort?: number; autoCloseTabs?: boolean; autoClickAllowDialog?: boolean }) => + request<{ bringToFront: boolean; remoteDebuggingPort: number; autoCloseTabs: boolean; autoClickAllowDialog: boolean; extensionBridgePort: number; extensionConnected: boolean }>('/settings/browser', { method: 'POST', body: JSON.stringify(settings) }), + testAutoClick: () => request<{ + checkResult: { platform: string; supported: boolean; accessibilityPermission: boolean; chromeRunning: boolean; binaryAvailable: boolean }; + openedAccessibilitySettings: boolean; + clickResult: 'success' | 'no_permission' | 'chrome_not_running' | 'unsupported' | 'error'; + pageLoaded: boolean; + pageTitle?: string; + error?: string; + }>('/settings/browser/test-auto-click', { method: 'POST' }), + downloadExtensionZip: () => { + const url = `${BASE}/settings/browser/extension.zip`; + const headers: Record = {}; + const token = localStorage.getItem('markus_token'); + if (token) headers['Authorization'] = `Bearer ${token}`; + return fetch(url, { headers }).then(r => { + if (!r.ok) throw new Error(`HTTP ${r.status}`); + return r.blob(); + }).then(blob => { + const a = document.createElement('a'); + a.href = URL.createObjectURL(blob); + a.download = 'markus-browser-extension.zip'; + a.click(); + URL.revokeObjectURL(a.href); + }); + }, + openExtensionsPage: () => request<{ ok: boolean }>('/settings/browser/open-extensions-page', { method: 'POST' }), getSearch: () => request<{ serper: { configured: boolean; preview: string }; brave: { configured: boolean; preview: string }; bocha: { configured: boolean; preview: string } }>('/settings/search'), updateSearch: (keys: { serperApiKey?: string; braveApiKey?: string; bochaApiKey?: string }) => request<{ serper: { configured: boolean; preview: string }; brave: { configured: boolean; preview: string }; bocha: { configured: boolean; preview: string } }>('/settings/search', { method: 'POST', body: JSON.stringify(keys) }), + getRemote: () => request('/settings/remote'), + enableRemote: () => request<{ ok: boolean; status: RemoteStatus }>('/settings/remote/enable', { method: 'POST' }), + disableRemote: () => request<{ ok: boolean }>('/settings/remote/disable', { method: 'POST' }), }, skills: { list: () => request<{ skills: Array<{ name: string; version: string; description?: string; author?: string; category?: string; tags?: string[]; tools?: Array<{ name: string; description: string }>; requiredPermissions?: string[]; type: 'builtin' | 'filesystem' | 'imported'; sourcePath?: string; agentIds: string[] }> }>('/skills'), @@ -1305,6 +1351,15 @@ export const api = { markAllRead: (userId: string) => request<{ success: boolean; count: number }>('/notifications/mark-all-read', { method: 'POST', body: JSON.stringify({ userId }) }), }, + // ─── Unread Tracking ─────────────────────────────────────────────── + unread: { + getCounts: () => request<{ counts: Record }>('/unread'), + markRead: (conversationKey: string, lastReadAt: string, lastReadId?: string) => + request<{ success: boolean }>('/unread/mark-read', { method: 'POST', body: JSON.stringify({ conversationKey, lastReadAt, lastReadId }) }), + markAllRead: () => + request<{ success: boolean }>('/unread/mark-all-read', { method: 'POST' }), + }, + // ─── Activity Feed ──────────────────────────────────────────────── activity: { list: (opts?: { limit?: number; type?: string }) => { @@ -1366,6 +1421,8 @@ export const api = { if (opts?.limit) params.set('limit', String(opts.limit)); return request<{ results: DeliverableInfo[]; total: number }>(`/deliverables?${params}`); }, + get: (id: string) => + request<{ deliverable: DeliverableInfo }>(`/deliverables/${id}`), create: (data: Partial) => request<{ deliverable: DeliverableInfo }>('/deliverables', { method: 'POST', body: JSON.stringify(data) }), update: (id: string, data: Partial) => diff --git a/packages/web-ui/src/components/BottomNav.tsx b/packages/web-ui/src/components/BottomNav.tsx index d7a34d81..f9864f34 100644 --- a/packages/web-ui/src/components/BottomNav.tsx +++ b/packages/web-ui/src/components/BottomNav.tsx @@ -12,6 +12,7 @@ interface Props { export function BottomNav({ currentPage, onNavigate, userId }: Props) { const { t } = useTranslation('nav'); const [unreadCount, setUnreadCount] = useState(0); + const [teamUnread, setTeamUnread] = useState(0); const fetchUnread = useCallback(async () => { try { @@ -25,7 +26,16 @@ export function BottomNav({ currentPage, onNavigate, userId }: Props) { const timer = setInterval(fetchUnread, 15000); const onChanged = () => fetchUnread(); window.addEventListener('markus:notifications-changed', onChanged); - return () => { clearInterval(timer); window.removeEventListener('markus:notifications-changed', onChanged); }; + const onTeamUnread = (e: Event) => { + const count = (e as CustomEvent).detail?.count ?? 0; + setTeamUnread(count); + }; + window.addEventListener('markus:team-unread-changed', onTeamUnread); + return () => { + clearInterval(timer); + window.removeEventListener('markus:notifications-changed', onChanged); + window.removeEventListener('markus:team-unread-changed', onTeamUnread); + }; }, [fetchUnread]); return ( @@ -33,6 +43,7 @@ export function BottomNav({ currentPage, onNavigate, userId }: Props) { {MOBILE_TABS.map(tab => { const isActive = tab.group.includes(currentPage); const isNotif = tab.id === PAGE.NOTIFICATIONS; + const isTeam = tab.id === PAGE.TEAM; return (
{t(tab.id)} diff --git a/packages/web-ui/src/components/ChatTeamSidebar.tsx b/packages/web-ui/src/components/ChatTeamSidebar.tsx index cca86642..57fb4291 100644 --- a/packages/web-ui/src/components/ChatTeamSidebar.tsx +++ b/packages/web-ui/src/components/ChatTeamSidebar.tsx @@ -1,6 +1,7 @@ import { useEffect, useMemo, useRef, useState, useCallback } from 'react'; import { useTranslation } from 'react-i18next'; import { useIsMobile } from '../hooks/useIsMobile.ts'; +import { MobileMenuButton } from './MobileMenuButton.tsx'; import { api, wsClient, type AgentInfo, type TeamInfo, type TeamMemberInfo, @@ -47,6 +48,8 @@ interface ChatTeamSidebarProps { onManageGroupMembers?: (channelKey: string) => void; /** Per-agent unread notification count (agentId → count) */ unreadByAgent?: Map; + /** Per-channel unread message count (channelKey → count) */ + unreadByChannel?: Record; width?: number; onResizeStart?: (e: React.MouseEvent) => void; hidden?: boolean; @@ -192,6 +195,7 @@ export function ChatTeamSidebar({ onRefreshTeams, onRefreshAgents, onRefreshHumans, onRefreshGroupChats, onViewProfile, onManageGroupMembers, unreadByAgent, + unreadByChannel, width, onResizeStart, hidden, initialLoading, }: ChatTeamSidebarProps) { @@ -655,7 +659,10 @@ export function ChatTeamSidebar({ const isDropTarget = isDragging && dragOverTeam === tid && dragAgent?.fromTeamId !== tid; const isHighlighted = highlightTeamId === tid; const isSelected = selectedTeamId === tid; - const teamUnread = unreadByAgent ? (agentsByTeam.byTeam.get(tid) ?? []).reduce((sum, a) => sum + (unreadByAgent.get(a.id) ?? 0), 0) : 0; + const agentUnread = unreadByAgent ? (agentsByTeam.byTeam.get(tid) ?? []).reduce((sum, a) => sum + (unreadByAgent.get(a.id) ?? 0), 0) : 0; + const teamGc = groupChats.find(gc => gc.type === 'team' && gc.teamId === tid); + const channelUnread = teamGc && unreadByChannel ? (unreadByChannel[teamGc.channelKey] ?? 0) : 0; + const teamUnread = agentUnread + channelUnread; return (
{/* Header with title + pause toggle */} -
+
+ {isMobile && }

{t('chat.title')}

{globalPaused === null ? ( @@ -1059,64 +1067,51 @@ export function ChatTeamSidebar({ ) : null} {/* Unmatched group chats (no matching team) */} - {groupChatsByTeam.unmatched.map(gc => ( - - ))} + {groupChatsByTeam.unmatched.map(gc => { + const chUnread = unreadByChannel?.[gc.channelKey] ?? 0; + return ( + + ); + })} - {isMobile ? ( - <> - {/* Mobile: original flat layout with nested agents */} + {/* Ungrouped agents listed individually */} + {agentsByTeam.ungrouped.length > 0 && ( +
+

{t('chat.agents')}

+ {agentsByTeam.ungrouped.map(a => renderAgentItem(a))} +
+ )} + {/* Teams as clickable rows (drill into L2 on mobile, expand on desktop) */} + {teams.length > 0 && ( +
+

{t('chat.teams')}

{teams.map(tm => { const agentList = agentsByTeam.byTeam.get(tm.id) ?? []; - if (agentList.length === 0 && (!tm.members || tm.members.length === 0)) return null; - return renderTeamSection(tm.id, tm, agentList, tm.name); + const memberCount = tm.members?.length || agentList.length; + return renderTeamRow(tm.id, tm, memberCount, tm.name); })} - {teams.filter(tm => { - const agentList = agentsByTeam.byTeam.get(tm.id) ?? []; - return agentList.length === 0 && (!tm.members || tm.members.length === 0); - }).map(tm => renderTeamSection(tm.id, tm, [], tm.name))} - {agentsByTeam.ungrouped.length > 0 && renderTeamSection('_ungrouped', null, agentsByTeam.ungrouped, t('chat.other'))} - - ) : ( - <> - {/* Desktop L1: ungrouped agents listed individually above teams */} - {agentsByTeam.ungrouped.length > 0 && ( -
-

{t('chat.agents')}

- {agentsByTeam.ungrouped.map(a => renderAgentItem(a))} -
- )} - {/* Desktop L1: simple clickable team rows */} - {teams.length > 0 && ( -
-

{t('chat.teams')}

- {teams.map(tm => { - const agentList = agentsByTeam.byTeam.get(tm.id) ?? []; - const memberCount = tm.members?.length || agentList.length; - return renderTeamRow(tm.id, tm, memberCount, tm.name); - })} -
- )} - +
)} {/* No teams — flat agent list */} diff --git a/packages/web-ui/src/components/MarkdownMessage.tsx b/packages/web-ui/src/components/MarkdownMessage.tsx index 30e05c74..8f51fe01 100644 --- a/packages/web-ui/src/components/MarkdownMessage.tsx +++ b/packages/web-ui/src/components/MarkdownMessage.tsx @@ -192,15 +192,17 @@ function localImageUrl(filePath: string): string { return `/api/files/image?path=${encodeURIComponent(filePath)}`; } -function MarkdownImage({ src, alt, onPreview, basePath }: { src: string; alt?: string; onPreview?: (src: string) => void; basePath?: string }) { - const [loaded, setLoaded] = useState(false); - const [error, setError] = useState(false); +const loadedImageCache = new Set(); +function MarkdownImage({ src, alt, onPreview, basePath }: { src: string; alt?: string; onPreview?: (src: string) => void; basePath?: string }) { const effectiveSrc = useMemo(() => { if (!isLocalImagePath(src)) return src; return localImageUrl(resolveImagePath(src, basePath)); }, [src, basePath]); + const [loaded, setLoaded] = useState(() => loadedImageCache.has(effectiveSrc)); + const [error, setError] = useState(false); + return ( {!loaded && !error && ( @@ -220,7 +222,7 @@ function MarkdownImage({ src, alt, onPreview, basePath }: { src: string; alt?: s src={effectiveSrc} alt={alt ?? ''} loading="lazy" - onLoad={() => setLoaded(true)} + onLoad={() => { loadedImageCache.add(effectiveSrc); setLoaded(true); }} onError={() => setError(true)} onClick={() => onPreview?.(effectiveSrc)} className="max-w-full h-auto rounded-lg cursor-pointer hover:opacity-90 transition-opacity my-1" diff --git a/packages/web-ui/src/components/MobileBuilderTabs.tsx b/packages/web-ui/src/components/MobileBuilderTabs.tsx index acce884c..93b98c39 100644 --- a/packages/web-ui/src/components/MobileBuilderTabs.tsx +++ b/packages/web-ui/src/components/MobileBuilderTabs.tsx @@ -1,15 +1,25 @@ -import { useState, useMemo } from 'react'; +import { useState, useMemo, useEffect } from 'react'; import { useTranslation } from 'react-i18next'; import { AgentBuilder } from '../pages/AgentBuilder.tsx'; import { TemplateMarketplace } from '../pages/TemplateMarketplace.tsx'; import { TeamsStore } from '../pages/TeamsStore.tsx'; import { SkillStore } from '../pages/SkillStore.tsx'; import { useSwipeTabs } from '../hooks/useSwipeTabs.ts'; +import { MobileMenuButton } from './MobileMenuButton.tsx'; import type { AuthUser } from '../api.ts'; const tabIds = ['builder', 'agents', 'teams', 'skills'] as const; type TabId = (typeof tabIds)[number]; +function getInitialTab(): TabId { + const stored = localStorage.getItem('markus_nav_storeTab'); + if (stored) { + localStorage.removeItem('markus_nav_storeTab'); + if (tabIds.includes(stored as TabId)) return stored as TabId; + } + return 'builder'; +} + export function MobileBuilderTabs({ authUser }: { authUser?: AuthUser }) { const { t } = useTranslation(['nav', 'common']); const tabs = useMemo(() => [ @@ -18,12 +28,24 @@ export function MobileBuilderTabs({ authUser }: { authUser?: AuthUser }) { { id: 'teams' as const, label: t('nav:tabs.teams') }, { id: 'skills' as const, label: t('nav:tabs.skills') }, ], [t]); - const [activeTab, setActiveTab] = useState('builder'); + const [activeTab, setActiveTab] = useState(getInitialTab); const swipe = useSwipeTabs(tabs, activeTab, setActiveTab); + useEffect(() => { + const handler = (e: Event) => { + const detail = (e as CustomEvent<{ page: string; params?: Record }>).detail; + if (detail.params?.storeTab && tabIds.includes(detail.params.storeTab as TabId)) { + setActiveTab(detail.params.storeTab as TabId); + } + }; + window.addEventListener('markus:navigate', handler); + return () => window.removeEventListener('markus:navigate', handler); + }, []); + return (
-
+
+ {tabs.map(tab => ( + ) : ( +
{t('common:loading')}
+ )} +
+ + {/* Navigation */} + + +
+ ); +} + +function DrawerNavItem({ icon, label, onClick }: { icon: React.ReactNode; label: string; onClick: () => void }) { + return ( + + ); +} + +export function openMobileDrawer() { + window.dispatchEvent(new CustomEvent('markus:open-drawer')); +} diff --git a/packages/web-ui/src/components/MobileMenuButton.tsx b/packages/web-ui/src/components/MobileMenuButton.tsx new file mode 100644 index 00000000..c18c79b0 --- /dev/null +++ b/packages/web-ui/src/components/MobileMenuButton.tsx @@ -0,0 +1,17 @@ +import { openMobileDrawer } from './MobileDrawer.tsx'; + +export function MobileMenuButton({ className = '' }: { className?: string }) { + return ( + + ); +} diff --git a/packages/web-ui/src/components/SearchModal.tsx b/packages/web-ui/src/components/SearchModal.tsx new file mode 100644 index 00000000..5e446087 --- /dev/null +++ b/packages/web-ui/src/components/SearchModal.tsx @@ -0,0 +1,433 @@ +import { useState, useCallback, useRef, useEffect, useMemo } from 'react'; +import { useTranslation } from 'react-i18next'; +import { api, type AgentInfo, type TaskInfo, type ProjectInfo, type DeliverableInfo, type RequirementInfo } from '../api.ts'; +import { navBus } from '../navBus.ts'; +import { PAGE, type PageId } from '../routes.ts'; +import { Avatar } from './Avatar.tsx'; + +type SearchCategory = 'all' | 'agents' | 'tasks' | 'requirements' | 'projects' | 'deliverables'; + +interface SearchResults { + agents: AgentInfo[]; + tasks: TaskInfo[]; + requirements: RequirementInfo[]; + projects: ProjectInfo[]; + deliverables: DeliverableInfo[]; +} + +interface FlatItem { + id: string; + type: 'agent' | 'task' | 'requirement' | 'project' | 'deliverable' | 'showMore'; + page: PageId; + params?: Record; + expandCategory?: SearchCategory; + totalCount?: number; +} + +const EMPTY: SearchResults = { agents: [], tasks: [], requirements: [], projects: [], deliverables: [] }; + +type SectionId = 'agents' | 'tasks' | 'requirements' | 'projects' | 'deliverables'; + +const PAGE_SECTION_ORDER: Record = { + team: ['agents', 'tasks', 'requirements', 'projects', 'deliverables'], + work: ['projects', 'requirements', 'tasks', 'agents', 'deliverables'], + deliverables: ['deliverables', 'agents', 'tasks', 'requirements', 'projects'], +}; +const DEFAULT_ORDER: SectionId[] = ['agents', 'tasks', 'requirements', 'projects', 'deliverables']; + +let _persistedQuery = ''; +let _persistedCategory: SearchCategory = 'all'; +let _persistedResults: SearchResults = EMPTY; +let _persistedSearched = false; + +export function SearchModal({ onClose, currentPage }: { onClose: () => void; currentPage?: string }) { + const { t } = useTranslation(['common', 'home', 'work']); + const [query, setQuery] = useState(_persistedQuery); + const [category, setCategory] = useState(_persistedCategory); + const [results, setResults] = useState(_persistedResults); + const [loading, setLoading] = useState(false); + const [searched, setSearched] = useState(_persistedSearched); + const [focusIdx, setFocusIdx] = useState(-1); + + useEffect(() => { _persistedQuery = query; }, [query]); + useEffect(() => { _persistedCategory = category; }, [category]); + useEffect(() => { _persistedResults = results; }, [results]); + useEffect(() => { _persistedSearched = searched; }, [searched]); + const inputRef = useRef(null); + const debounceRef = useRef | undefined>(undefined); + const backdropRef = useRef(null); + const listRef = useRef(null); + + useEffect(() => { + setTimeout(() => { inputRef.current?.focus(); inputRef.current?.select(); }, 100); + }, []); + + const doSearch = useCallback(async (q: string) => { + if (!q.trim()) { setResults(EMPTY); setSearched(false); setFocusIdx(-1); return; } + setLoading(true); + setSearched(true); + setFocusIdx(-1); + const lower = q.toLowerCase(); + try { + const [agentsRes, tasksRes, requirementsRes, projectsRes, deliverablesRes] = await Promise.allSettled([ + api.agents.list(), + api.tasks.list({ search: q, pageSize: 20 }), + api.requirements.list(), + api.projects.list(), + api.deliverables.search({ q, limit: 20 }), + ]); + const agents = agentsRes.status === 'fulfilled' + ? agentsRes.value.agents.filter(a => a.name?.toLowerCase().includes(lower) || a.role?.toLowerCase().includes(lower)) + : []; + const tasks = tasksRes.status === 'fulfilled' ? tasksRes.value.tasks : []; + const requirements = requirementsRes.status === 'fulfilled' + ? requirementsRes.value.requirements.filter(r => r.title?.toLowerCase().includes(lower) || r.description?.toLowerCase().includes(lower)) + : []; + const projects = projectsRes.status === 'fulfilled' + ? projectsRes.value.projects.filter(p => p.name?.toLowerCase().includes(lower) || p.description?.toLowerCase().includes(lower)) + : []; + const deliverables = deliverablesRes.status === 'fulfilled' ? deliverablesRes.value.results : []; + setResults({ agents, tasks, requirements, projects, deliverables }); + } catch { + setResults(EMPTY); + } finally { + setLoading(false); + } + }, []); + + const handleInput = (value: string) => { + setQuery(value); + if (debounceRef.current) clearTimeout(debounceRef.current); + debounceRef.current = setTimeout(() => doSearch(value), 400); + }; + + const navigate = useCallback((page: PageId, params?: Record) => { + onClose(); + navBus.navigate(page, params); + }, [onClose]); + + const categories: { id: SearchCategory; label: string }[] = [ + { id: 'all', label: t('common:all', { defaultValue: '全部' }) }, + { id: 'agents', label: t('common:agents', { defaultValue: '智能体' }) }, + { id: 'tasks', label: t('common:tasks', { defaultValue: '任务' }) }, + { id: 'requirements', label: t('common:requirements', { defaultValue: '需求' }) }, + { id: 'projects', label: t('common:projects', { defaultValue: '项目' }) }, + { id: 'deliverables', label: t('common:deliverables', { defaultValue: '交付物' }) }, + ]; + + const sectionOrder = useMemo(() => PAGE_SECTION_ORDER[currentPage || ''] || DEFAULT_ORDER, [currentPage]); + + const flatItems = useMemo(() => { + const items: FlatItem[] = []; + const limit = category === 'all' ? 8 : undefined; + const addSection = (section: SectionId) => { + if (category !== 'all' && category !== section) return; + const arr = results[section === 'agents' ? 'agents' : section === 'tasks' ? 'tasks' : section === 'requirements' ? 'requirements' : section === 'projects' ? 'projects' : 'deliverables']; + const sliced = limit ? arr.slice(0, limit) : arr; + for (const item of sliced) { + switch (section) { + case 'agents': items.push({ id: item.id, type: 'agent', page: PAGE.TEAM, params: { agentId: item.id } }); break; + case 'tasks': items.push({ id: item.id, type: 'task', page: PAGE.WORK, params: { openTask: item.id } }); break; + case 'requirements': items.push({ id: item.id, type: 'requirement', page: PAGE.WORK, params: { openRequirement: item.id } }); break; + case 'projects': items.push({ id: item.id, type: 'project', page: PAGE.WORK, params: { projectId: item.id } }); break; + case 'deliverables': items.push({ id: item.id, type: 'deliverable', page: PAGE.DELIVERABLES, params: { openDeliverable: item.id } }); break; + } + } + if (limit && arr.length > limit) { + items.push({ id: `more_${section}`, type: 'showMore', page: '' as PageId, expandCategory: section as SearchCategory, totalCount: arr.length }); + } + }; + for (const s of sectionOrder) addSection(s); + return items; + }, [results, category, sectionOrder]); + + const openItem = useCallback((item: FlatItem) => { + if (item.type === 'showMore' && item.expandCategory) { + setCategory(item.expandCategory); + setFocusIdx(-1); + return; + } + navigate(item.page, item.params); + }, [navigate]); + + useEffect(() => { + const el = listRef.current; + if (!el || focusIdx < 0) return; + const target = el.querySelector(`[data-idx="${focusIdx}"]`); + if (target) target.scrollIntoView({ block: 'nearest' }); + }, [focusIdx]); + + const handleKeyDown = useCallback((e: KeyboardEvent) => { + if (e.key === 'Escape') { onClose(); return; } + if (e.key === 'Tab') { + e.preventDefault(); + const catIds = categories.map(c => c.id); + const curIdx = catIds.indexOf(category); + const next = e.shiftKey + ? (curIdx <= 0 ? catIds.length - 1 : curIdx - 1) + : (curIdx >= catIds.length - 1 ? 0 : curIdx + 1); + setCategory(catIds[next]); + setFocusIdx(-1); + return; + } + const isDown = (e.ctrlKey && e.key === 'n') || e.key === 'ArrowDown'; + const isUp = (e.ctrlKey && e.key === 'p') || e.key === 'ArrowUp'; + if (isDown || isUp) { + e.preventDefault(); + setFocusIdx(prev => { + const max = flatItems.length - 1; + if (max < 0) return -1; + if (isDown) return prev >= max ? 0 : prev + 1; + return prev <= 0 ? max : prev - 1; + }); + return; + } + if (e.key === 'Enter') { + e.preventDefault(); + if (focusIdx >= 0 && focusIdx < flatItems.length) { + openItem(flatItems[focusIdx]); + } + } + }, [onClose, flatItems, focusIdx, openItem, categories, category]); + + useEffect(() => { + document.addEventListener('keydown', handleKeyDown); + return () => document.removeEventListener('keydown', handleKeyDown); + }, [handleKeyDown]); + + const hasResults = flatItems.length > 0; + + let itemCounter = 0; + + return ( +
{ if (e.target === backdropRef.current) onClose(); }} + > +
+ {/* Search input */} +
+
+ + handleInput(e.target.value)} + placeholder={t('common:search')} + className="w-full bg-surface-elevated border border-border-default rounded-xl pl-9 pr-9 py-2.5 text-sm text-fg-primary placeholder:text-fg-tertiary focus:border-brand-500 focus:outline-none transition-colors" + /> + {query && ( + + )} +
+
+ {categories.map(cat => ( + + ))} +
+
+ + {/* Results */} +
+ {loading && ( +
+
{t('common:loading')}
+
+ )} + + {!loading && searched && !hasResults && ( +
+ +

{t('common:noResults', { defaultValue: '没有找到相关结果' })}

+
+ )} + + {!loading && hasResults && ( +
+ {sectionOrder.map(section => { + if (category !== 'all' && category !== section) return null; + const limit = category === 'all' ? 8 : undefined; + const showMoreBtn = (cat: SearchCategory) => { + const idx = itemCounter++; + const total = cat === 'agents' ? results.agents.length + : cat === 'tasks' ? results.tasks.length + : cat === 'requirements' ? results.requirements.length + : cat === 'projects' ? results.projects.length + : results.deliverables.length; + return ( + + ); + }; + if (section === 'agents' && results.agents.length > 0) return ( +
+

{t('common:agents', { defaultValue: '智能体' })}

+
+ {(limit ? results.agents.slice(0, limit) : results.agents).map(agent => { + const idx = itemCounter++; + return ( + + ); + })} +
+ {limit && results.agents.length > limit && showMoreBtn('agents')} +
+ ); + if (section === 'tasks' && results.tasks.length > 0) return ( +
+

{t('common:tasks', { defaultValue: '任务' })}

+
+ {(limit ? results.tasks.slice(0, limit) : results.tasks).map(task => { + const idx = itemCounter++; + return ( + + ); + })} +
+ {limit && results.tasks.length > limit && showMoreBtn('tasks')} +
+ ); + if (section === 'requirements' && results.requirements.length > 0) return ( +
+

{t('common:requirements', { defaultValue: '需求' })}

+
+ {(limit ? results.requirements.slice(0, limit) : results.requirements).map(req => { + const idx = itemCounter++; + return ( + + ); + })} +
+ {limit && results.requirements.length > limit && showMoreBtn('requirements')} +
+ ); + if (section === 'projects' && results.projects.length > 0) return ( +
+

{t('common:projects', { defaultValue: '项目' })}

+
+ {(limit ? results.projects.slice(0, limit) : results.projects).map(proj => { + const idx = itemCounter++; + return ( + + ); + })} +
+ {limit && results.projects.length > limit && showMoreBtn('projects')} +
+ ); + if (section === 'deliverables' && results.deliverables.length > 0) return ( +
+

{t('common:deliverables', { defaultValue: '交付物' })}

+
+ {(limit ? results.deliverables.slice(0, limit) : results.deliverables).map(d => { + const idx = itemCounter++; + return ( + + ); + })} +
+ {limit && results.deliverables.length > limit && showMoreBtn('deliverables')} +
+ ); + return null; + })} +
+ )} + + {!loading && !searched && ( +
+ +

{t('common:searchHint', { defaultValue: '输入关键词搜索' })}

+
+ )} +
+ + {/* Footer */} +
+ {navigator.platform.toUpperCase().includes('MAC') ? 'Cmd+P' : 'Ctrl+P'} {t('common:toggle', { defaultValue: '唤起/关闭' })} + Tab {t('common:switchTab', { defaultValue: '切换分类' })} + ↑↓ {t('common:navigate', { defaultValue: '导航' })} + Enter {t('common:open', { defaultValue: '打开' })} + Esc {t('common:close', { defaultValue: '关闭' })} +
+
+
+ ); +} + +function StatusDot({ status }: { status: string }) { + const color = status === 'active' || status === 'idle' ? 'bg-green-500' + : status === 'working' || status === 'busy' ? 'bg-blue-500' + : status === 'paused' ? 'bg-amber-500' + : 'bg-gray-400'; + return ; +} + +function TaskStatusIcon({ status }: { status: string }) { + const color = status === 'completed' ? 'text-green-500 bg-green-500/15' + : status === 'in_progress' ? 'text-blue-500 bg-blue-500/15' + : status === 'pending' ? 'text-amber-500 bg-amber-500/15' + : 'text-fg-tertiary bg-surface-elevated'; + return ( +
+ +
+ ); +} diff --git a/packages/web-ui/src/hooks/useUnreadCounts.ts b/packages/web-ui/src/hooks/useUnreadCounts.ts new file mode 100644 index 00000000..3c6d77be --- /dev/null +++ b/packages/web-ui/src/hooks/useUnreadCounts.ts @@ -0,0 +1,126 @@ +import { useState, useEffect, useCallback, useRef, useMemo } from 'react'; +import { api, wsClient } from '../api.ts'; + +const POLL_INTERVAL_MS = 60_000; + +let _globalCounts: Record = {}; +const _listeners = new Set<() => void>(); + +function notify() { + for (const fn of _listeners) fn(); +} + +export function useUnreadCounts() { + const [counts, setCounts] = useState>(_globalCounts); + const pollRef = useRef | undefined>(undefined); + + const refresh = useCallback(async () => { + try { + const resp = await api.unread.getCounts(); + _globalCounts = resp.counts ?? {}; + setCounts(_globalCounts); + notify(); + } catch { /* silent */ } + }, []); + + const markRead = useCallback(async (conversationKey: string) => { + const ts = new Date().toISOString(); + delete _globalCounts[conversationKey]; + setCounts({ ..._globalCounts }); + notify(); + try { + await api.unread.markRead(conversationKey, ts); + } catch { /* silent */ } + }, []); + + const markAllRead = useCallback(async () => { + _globalCounts = {}; + setCounts({}); + notify(); + try { + await api.unread.markAllRead(); + } catch { /* silent */ } + }, []); + + useEffect(() => { + refresh(); + pollRef.current = setInterval(refresh, POLL_INTERVAL_MS); + + const unsub = wsClient.on('chat:unread_update', (event) => { + const key = (event.payload as { conversationKey?: string })?.conversationKey; + if (key) { + _globalCounts[key] = (_globalCounts[key] ?? 0) + 1; + setCounts({ ..._globalCounts }); + notify(); + } + }); + + const listener = () => setCounts({ ..._globalCounts }); + _listeners.add(listener); + + return () => { + if (pollRef.current) clearInterval(pollRef.current); + unsub(); + _listeners.delete(listener); + }; + }, [refresh]); + + const totalUnread = useMemo(() => { + return Object.values(counts).reduce((sum, n) => sum + n, 0); + }, [counts]); + + const getSessionUnread = useCallback((sessionId: string): number => { + return counts[`session:${sessionId}`] ?? 0; + }, [counts]); + + const getChannelUnread = useCallback((channelKey: string): number => { + return counts[`channel:${channelKey}`] ?? 0; + }, [counts]); + + return { counts, totalUnread, getSessionUnread, getChannelUnread, markRead, markAllRead, refresh }; +} + +/** + * Get unread count for a specific agent by summing all session:* entries + * that belong to sessions of that agent. + * Since session keys encode the session ID (not the agent ID), the caller + * must provide a mapping of sessionId -> agentId. + */ +export function useAgentUnread( + agentSessionMap: Map, + counts: Record +): Map { + return useMemo(() => { + const result = new Map(); + for (const [key, count] of Object.entries(counts)) { + if (key.startsWith('session:')) { + const sessionId = key.slice('session:'.length); + const agentId = agentSessionMap.get(sessionId); + if (agentId) { + result.set(agentId, (result.get(agentId) ?? 0) + count); + } + } + } + return result; + }, [agentSessionMap, counts]); +} + +/** + * Get unread for a team by summing its team channel + all member agent sessions. + */ +export function getTeamUnread( + teamId: string, + teamAgentIds: string[], + teamChannelKey: string | undefined, + agentUnreads: Map, + counts: Record +): number { + let total = 0; + if (teamChannelKey) { + total += counts[`channel:${teamChannelKey}`] ?? 0; + } + for (const agentId of teamAgentIds) { + total += agentUnreads.get(agentId) ?? 0; + } + return total; +} diff --git a/packages/web-ui/src/index.css b/packages/web-ui/src/index.css index a6bda948..932ff531 100644 --- a/packages/web-ui/src/index.css +++ b/packages/web-ui/src/index.css @@ -270,3 +270,18 @@ body { z-index: -1; pointer-events: none; } + +@keyframes fadeIn { + from { opacity: 0; } + to { opacity: 1; } +} +@keyframes slideInLeft { + from { transform: translateX(-100%); } + to { transform: translateX(0); } +} +.animate-fadeIn { + animation: fadeIn 0.2s ease-out; +} +.animate-slideInLeft { + animation: slideInLeft 0.25s ease-out; +} diff --git a/packages/web-ui/src/locales/en/common.json b/packages/web-ui/src/locales/en/common.json index 721498b2..100ed7c6 100644 --- a/packages/web-ui/src/locales/en/common.json +++ b/packages/web-ui/src/locales/en/common.json @@ -12,6 +12,8 @@ "creating": "Creating...", "refresh": "Refresh", "back": "Back", + "home": "Home", + "reports": "Reports", "next": "Next", "skip": "Skip", "dismiss": "Dismiss", diff --git a/packages/web-ui/src/locales/en/settings.json b/packages/web-ui/src/locales/en/settings.json index 9a775c08..38147aa1 100644 --- a/packages/web-ui/src/locales/en/settings.json +++ b/packages/web-ui/src/locales/en/settings.json @@ -1,5 +1,15 @@ { "title": "Settings", + "nav": { + "appearance": "Appearance", + "providers": "Model Providers", + "execution": "Agent & Pipeline", + "browser": "Browser Automation", + "search": "Web Search", + "storage": "Data & Storage", + "users": "User Management", + "remote": "Remote Access" + }, "appearance": { "title": "Appearance", "theme": "Theme", @@ -39,6 +49,16 @@ "title": "Manual Configuration", "description": "Edit ~/.markus/markus.json directly, then restart the server." }, + "browserExtension": { + "step": "5", + "title": "Install Browser Automation Extension", + "description": "Let AI agents control Chrome to browse, test apps, and inspect pages — no permission dialog each time.", + "notChrome": "Your current browser is not Chrome. Open Markus in Chrome to install the extension.", + "alreadyConnected": "Extension is already connected.", + "downloadBtn": "Download Extension", + "openChromeBtn": "Open Extensions Page", + "loadHint": "After unzipping, go to Chrome Extensions page → enable \"Developer mode\" → \"Load unpacked\" → select the unzipped folder" + }, "applyProviders": "Apply {{count}} Provider(s)", "foundPrefix": "Found:" }, @@ -242,7 +262,49 @@ "tabsForeground": "Tabs will be brought to foreground", "tabsBackground": "Tabs will stay in background", "usingPort": "Using persistent connection on port {{port}} (no more permission dialogs)", - "usingAutoConnect": "Using auto-connect mode (permission dialog required on each connection)" + "usingAutoConnect": "Using auto-connect mode (permission dialog required on each connection)", + "autoClickAllowDialog": "Auto-Allow Chrome Debugging Dialog", + "autoClickAllowDialogDesc": "Chrome shows an \"Allow remote debugging?\" dialog each time Markus connects. When enabled, Markus auto-clicks \"Allow\" via OS accessibility APIs.", + "autoClickAllowDialogMacNote": "macOS: Grant Accessibility permission in System Settings > Privacy & Security > Accessibility for the app running Markus (Markus.app / Terminal / iTerm).", + "autoClickAllowDialogWinNote": "Windows: No additional permissions needed. Uses built-in UI Automation.", + "autoClickAllowDialogLinuxNote": "Linux: Not yet supported. Use Remote Debugging Port (above) as an alternative.", + "autoClickAllowDialogSupported": "Supported on this platform", + "autoClickAllowDialogUnsupported": "Not supported on this platform", + "autoClickTest": "Test", + "autoClickTesting": "Testing...", + "autoClickTestSuccess": "Auto-click verified! Successfully connected to Chrome and loaded a test page ({{title}}).", + "autoClickTestNoPermission": "Accessibility permission not granted. The system settings page has been opened — please add this app to the list.", + "autoClickTestChromeNotRunning": "Chrome is not running. Please start Chrome first.", + "autoClickTestUnsupported": "Auto-click is not supported on this platform. Use Remote Debugging Port instead.", + "autoClickTestError": "Test failed: {{error}}", + "modeExtension": "Chrome Extension (Active)", + "modeExtensionDesc": "Browser tools route through the extension — no debugging dialog, no extra permissions, works when screen is locked.", + "modeDebuggingPort": "Remote Debugging Port :{{port}} (Active)", + "modeDebuggingPortDesc": "Connected via persistent debugging port — no permission dialog on each connection.", + "modeAutoClick": "Auto-Connect + Auto-Click (Active)", + "modeAutoClickDesc": "Using chrome-devtools-mcp with OS auto-click to handle the debugging dialog. May not work when screen is locked.", + "modeManual": "Auto-Connect (Manual Approval Required)", + "modeManualDesc": "Chrome will show a permission dialog each time an agent connects. You must click Allow manually.", + "modeRecommended": "recommended", + "generalSettings": "General", + "fallbackModes": "Connection Mode (Fallback)", + "extensionStatus": "Chrome Extension", + "extensionStatusDesc": "Enables browser automation without the debugging dialog. No Accessibility permissions needed, works even when screen is locked.", + "extensionConnected": "Connected", + "extensionDisconnected": "Not Connected", + "extensionActiveNote": "Extension active — browser tools route through extension (no debugging dialog)", + "extensionSetupTitle": "Install Chrome Extension", + "extensionStep1": "Download & Unzip", + "extensionStep1Desc": "Download the zip and extract it to any location (e.g. Desktop)", + "extensionStep1Btn": "Download markus-browser-extension.zip", + "extensionStep1Downloading": "Preparing download...", + "extensionStep2": "Open Chrome Extensions", + "extensionStep2Desc": "Open the extensions management page in Chrome", + "extensionStep2Btn": "Open chrome://extensions", + "extensionStep3": "Load Extension", + "extensionStep3Desc": "Enable \"Developer mode\" (top right toggle), click \"Load unpacked\", and select the folder extracted in step 1", + "extensionDownloadError": "Failed to download extension", + "extensionOpenError": "Failed to open Chrome extensions page" }, "dataStorage": { "title": "Data & Storage", @@ -271,6 +333,31 @@ "oauthConnected": "Connected to {{provider}} via OAuth", "authProfileDeleted": "Auth profile deleted", "browserOpened": "Browser opened for authorization. Complete the login in the browser window...", + "remoteAccess": { + "title": "Remote Access", + "description": "Access this Markus instance from your phone or any device via WebRTC P2P connection through Markus Hub.", + "loginRequired": "Sign in to Markus Hub to enable remote access.", + "signIn": "Sign In", + "signedInAs": "Signed in as", + "enabled": "Remote access enabled", + "disabled": "Remote access disabled", + "connecting": "Connecting to signal server...", + "registering": "Registering with Hub...", + "connected": "Connected", + "disconnected": "Disconnected", + "peerCount_one": "{{count}} peer", + "peerCount_other": "{{count}} peers", + "url": "Remote URL", + "copy": "Copy", + "copied": "Copied!", + "qrCode": "QR Code", + "scanQr": "Scan with your phone camera to connect", + "enableFailed": "Failed to enable remote access", + "disableFailed": "Failed to disable remote access", + "connectedPeers": "Connected Peers", + "connectedSince": "Connected since {{time}}", + "peerConnecting": "Connecting" + }, "userManagement": { "title": "User Management", "name": "Name", diff --git a/packages/web-ui/src/locales/zh-CN/common.json b/packages/web-ui/src/locales/zh-CN/common.json index c46d0af6..9c1f1124 100644 --- a/packages/web-ui/src/locales/zh-CN/common.json +++ b/packages/web-ui/src/locales/zh-CN/common.json @@ -12,6 +12,8 @@ "creating": "正在创建...", "refresh": "刷新", "back": "返回", + "home": "首页", + "reports": "报告", "next": "下一步", "skip": "跳过", "dismiss": "关闭", diff --git a/packages/web-ui/src/locales/zh-CN/settings.json b/packages/web-ui/src/locales/zh-CN/settings.json index 298d4e1b..6aacbc13 100644 --- a/packages/web-ui/src/locales/zh-CN/settings.json +++ b/packages/web-ui/src/locales/zh-CN/settings.json @@ -1,5 +1,15 @@ { "title": "设置", + "nav": { + "appearance": "外观", + "providers": "模型配置", + "execution": "智能体与流水线", + "browser": "浏览器自动化", + "search": "网络搜索", + "storage": "数据与存储", + "users": "用户管理", + "remote": "远程访问" + }, "appearance": { "title": "外观", "theme": "主题", @@ -39,6 +49,16 @@ "title": "手动配置", "description": "直接编辑 ~/.markus/markus.json，然后重启服务器。" }, + "browserExtension": { + "step": "5", + "title": "安装浏览器自动化扩展", + "description": "让 AI 智能体能直接操控 Chrome 浏览网页、测试应用，无需每次弹窗授权。", + "notChrome": "当前浏览器不是 Chrome，请在 Chrome 中打开 Markus 以安装扩展。", + "alreadyConnected": "扩展已连接，无需操作。", + "downloadBtn": "下载扩展", + "openChromeBtn": "打开扩展页面", + "loadHint": "解压后，在 Chrome 扩展页面开启「开发者模式」→「加载已解压的扩展程序」→ 选择解压文件夹" + }, "applyProviders": "应用 {{count}} 个提供商", "foundPrefix": "已找到：" }, @@ -242,7 +262,49 @@ "tabsForeground": "标签页将置于前台", "tabsBackground": "标签页将保留在后台", "usingPort": "正在使用端口 {{port}} 的持久连接（不再出现权限对话框）", - "usingAutoConnect": "正在使用自动连接模式（每次连接都需要权限对话框）" + "usingAutoConnect": "正在使用自动连接模式（每次连接都需要权限对话框）", + "autoClickAllowDialog": "自动允许 Chrome 调试弹窗", + "autoClickAllowDialogDesc": "Chrome 每次被 Markus 连接时都会弹出「允许远程调试？」对话框。开启后，Markus 通过操作系统辅助功能 API 自动点击「允许」按钮。", + "autoClickAllowDialogMacNote": "macOS：需在「系统设置 → 隐私与安全性 → 辅助功能」中授予运行 Markus 的应用（Markus.app / Terminal / iTerm）权限。", + "autoClickAllowDialogWinNote": "Windows：无需额外权限，使用内置 UI Automation。", + "autoClickAllowDialogLinuxNote": "Linux：暂不支持。请使用上方「远程调试端口」作为替代方案。", + "autoClickAllowDialogSupported": "当前平台支持", + "autoClickAllowDialogUnsupported": "当前平台暂不支持", + "autoClickTest": "测试", + "autoClickTesting": "测试中...", + "autoClickTestSuccess": "自动点击验证通过！已成功连接 Chrome 并加载测试页面（{{title}}）。", + "autoClickTestNoPermission": "辅助功能权限未授予。已打开系统设置页面，请将本应用添加到列表中。", + "autoClickTestChromeNotRunning": "Chrome 未运行。请先启动 Chrome。", + "autoClickTestUnsupported": "当前平台不支持自动点击。请使用「远程调试端口」作为替代方案。", + "autoClickTestError": "测试失败：{{error}}", + "modeExtension": "Chrome 扩展（已激活）", + "modeExtensionDesc": "浏览器工具通过扩展运行 — 无调试弹窗、无需额外权限、锁屏可用。", + "modeDebuggingPort": "远程调试端口 :{{port}}（已激活）", + "modeDebuggingPortDesc": "通过持久调试端口连接 — 每次连接无需权限弹窗。", + "modeAutoClick": "自动连接 + 自动点击（已激活）", + "modeAutoClickDesc": "使用 chrome-devtools-mcp 并通过系统 API 自动点击调试弹窗。锁屏时可能不工作。", + "modeManual": "自动连接（需手动允许）", + "modeManualDesc": "每次 Agent 连接浏览器时，Chrome 会弹出权限对话框，需要手动点击「允许」。", + "modeRecommended": "推荐", + "generalSettings": "通用设置", + "fallbackModes": "连接方式（备选）", + "extensionStatus": "Chrome 扩展", + "extensionStatusDesc": "无需调试弹窗即可实现浏览器自动化。无需辅助功能权限，锁屏状态下也能正常工作。", + "extensionConnected": "已连接", + "extensionDisconnected": "未连接", + "extensionActiveNote": "扩展已激活 — 浏览器工具通过扩展运行（无调试弹窗）", + "extensionSetupTitle": "安装 Chrome 扩展", + "extensionStep1": "下载并解压扩展", + "extensionStep1Desc": "下载压缩包后，解压到任意位置（如桌面）", + "extensionStep1Btn": "下载 markus-browser-extension.zip", + "extensionStep1Downloading": "准备下载...", + "extensionStep2": "打开 Chrome 扩展页面", + "extensionStep2Desc": "在 Chrome 中打开扩展管理页面", + "extensionStep2Btn": "打开 chrome://extensions", + "extensionStep3": "加载扩展", + "extensionStep3Desc": "开启右上角的「开发者模式」开关，点击「加载已解压的扩展程序」，选择第 1 步解压出来的文件夹", + "extensionDownloadError": "下载扩展失败", + "extensionOpenError": "打开 Chrome 扩展页面失败" }, "dataStorage": { "title": "数据与存储", @@ -271,6 +333,31 @@ "oauthConnected": "已通过 OAuth 连接到 {{provider}}", "authProfileDeleted": "认证配置已删除", "browserOpened": "已打开浏览器以完成授权。请在浏览器窗口中完成登录…", + "remoteAccess": { + "title": "远程访问", + "description": "通过 Markus Hub 的 WebRTC P2P 连接，从手机或任何设备访问此 Markus 实例。", + "loginRequired": "请先登录 Markus Hub 以启用远程访问。", + "signIn": "登录", + "signedInAs": "已登录：", + "enabled": "远程访问已启用", + "disabled": "远程访问已禁用", + "connecting": "正在连接信令服务器…", + "registering": "正在注册到 Hub…", + "connected": "已连接", + "disconnected": "已断开", + "peerCount_one": "{{count}} 个对等节点", + "peerCount_other": "{{count}} 个对等节点", + "url": "远程访问地址", + "copy": "复制", + "copied": "已复制！", + "qrCode": "二维码", + "scanQr": "用手机相机扫描以连接", + "enableFailed": "启用远程访问失败", + "disableFailed": "禁用远程访问失败", + "connectedPeers": "已连接的设备", + "connectedSince": "连接于 {{time}}", + "peerConnecting": "连接中" + }, "userManagement": { "title": "用户管理", "name": "姓名", diff --git a/packages/web-ui/src/pages/Deliverables.tsx b/packages/web-ui/src/pages/Deliverables.tsx index 3c3ae5cc..2085227c 100644 --- a/packages/web-ui/src/pages/Deliverables.tsx +++ b/packages/web-ui/src/pages/Deliverables.tsx @@ -8,6 +8,7 @@ import { ArtifactPreview, type BuilderMode } from '../components/BuilderArtifact import { navBus } from '../navBus.ts'; import { PAGE } from '../routes.ts'; import { useIsMobile } from '../hooks/useIsMobile.ts'; +import { MobileMenuButton } from '../components/MobileMenuButton.tsx'; import { useResizablePanel } from '../hooks/useResizablePanel.ts'; import { useSwipeTabs } from '../hooks/useSwipeTabs.ts'; @@ -166,6 +167,52 @@ export function DeliverablesPage({ authUser: _authUser }: { authUser?: AuthUser return () => { unsub1(); unsub2(); unsub3(); }; }, [refresh]); + // Handle deep navigation to a specific deliverable + const pendingOpenRef = useRef(null); + const itemsRef = useRef(items); + itemsRef.current = items; + + const openDeliverableById = useCallback((id: string) => { + const showDetail = (item: DeliverableInfo) => { + setSelected(item); + if (isMobile) { + setMobileShowDetail(true); + history.pushState({ mobileDetail: PAGE.DELIVERABLES }, '', window.location.hash); + } + }; + const found = itemsRef.current.find(d => d.id === id); + if (found) { showDetail(found); return; } + api.deliverables.get(id).then(r => { if (r.deliverable) showDetail(r.deliverable); }).catch(() => {}); + }, [isMobile]); + + useEffect(() => { + const navId = localStorage.getItem('markus_nav_openDeliverable'); + if (navId) { + localStorage.removeItem('markus_nav_openDeliverable'); + if (itemsRef.current.length > 0) { + openDeliverableById(navId); + } else { + pendingOpenRef.current = navId; + } + } + const handler = (e: Event) => { + const detail = (e as CustomEvent).detail; + if (detail?.params?.openDeliverable) { + localStorage.removeItem('markus_nav_openDeliverable'); + openDeliverableById(detail.params.openDeliverable); + } + }; + window.addEventListener('markus:navigate', handler); + return () => window.removeEventListener('markus:navigate', handler); + }, [openDeliverableById]); + + useEffect(() => { + const id = pendingOpenRef.current; + if (!id || items.length === 0) return; + pendingOpenRef.current = null; + openDeliverableById(id); + }, [items, openDeliverableById]); + const checkNeedMore = useCallback(() => { const el = listRef.current; if (!el || loading || loadingMore || items.length >= totalCount) return; @@ -340,11 +387,14 @@ export function DeliverablesPage({ authUser: _authUser }: { authUser?: AuthUser
-
-

- {t('title')}{totalCount > 0 && ({totalCount})} -

- +
+
+ {isMobile && } +

+ {t('title')}{totalCount > 0 && ({totalCount})} +

+
+
([]); const [teams, setTeams] = useState([]); const [board, setBoard] = useState>({}); @@ -39,7 +42,18 @@ export function HomePage({ authUser }: { authUser?: { id: string; name: string; const [pendingReqs, setPendingReqs] = useState([]); const [storageInfo, setStorageInfo] = useState(null); const [showDeployChoice, setShowDeployChoice] = useState(false); + const [showCreateMenu, setShowCreateMenu] = useState(false); const [showRankingModal, setShowRankingModal] = useState(false); + const createMenuRef = useRef(null); + + useEffect(() => { + if (!showCreateMenu) return; + const handler = (e: MouseEvent) => { + if (createMenuRef.current && !createMenuRef.current.contains(e.target as Node)) setShowCreateMenu(false); + }; + document.addEventListener('mousedown', handler); + return () => document.removeEventListener('mousedown', handler); + }, [showCreateMenu]); const refresh = () => { api.agents.list().then(d => setAgents(d.agents)).catch(() => {}); @@ -120,19 +134,110 @@ export function HomePage({ authUser }: { authUser?: { id: string; name: string;
{/* Header */}
-
+
+ {isMobile && }

{t('title')}

{t('subtitle')}

- + {isMobile ? ( +
+ +
+ + {showCreateMenu && ( +
+
+ + + +
+ +
+
+ )} +
+
+ ) : ( +
+ +
+ + {showCreateMenu && ( +
+
+ + + +
+ +
+
+ )} +
+
+ )}
diff --git a/packages/web-ui/src/pages/Notifications.tsx b/packages/web-ui/src/pages/Notifications.tsx index d9821453..8a836871 100644 --- a/packages/web-ui/src/pages/Notifications.tsx +++ b/packages/web-ui/src/pages/Notifications.tsx @@ -1,12 +1,16 @@ import { useTranslation } from 'react-i18next'; import { NotificationBell } from '../components/NotificationBell.tsx'; +import { MobileMenuButton } from '../components/MobileMenuButton.tsx'; +import { useIsMobile } from '../hooks/useIsMobile.ts'; export function NotificationsPage({ authUser }: { authUser?: { id: string; name: string; role: string; orgId: string } }) { const { t } = useTranslation('nav'); + const isMobile = useIsMobile(); return (
-
+
+ {isMobile && }

{t('notifications')}

diff --git a/packages/web-ui/src/pages/Reports.tsx b/packages/web-ui/src/pages/Reports.tsx index dd942b65..5b432f83 100644 --- a/packages/web-ui/src/pages/Reports.tsx +++ b/packages/web-ui/src/pages/Reports.tsx @@ -3,6 +3,8 @@ import { useTranslation } from 'react-i18next'; import { api, type ReportInfo, type ReportFeedbackInfo, type AgentUsageInfo, type AuthUser } from '../api.ts'; import { navBus } from '../navBus.ts'; import { PAGE } from '../routes.ts'; +import { MobileMenuButton } from '../components/MobileMenuButton.tsx'; +import { useIsMobile } from '../hooks/useIsMobile.ts'; type Period = 'daily' | 'weekly' | 'monthly'; interface ReportsPageProps { authUser?: AuthUser } @@ -36,6 +38,7 @@ function formatBytes(b: number): string { export function ReportsPage({ authUser }: ReportsPageProps) { const { t } = useTranslation(['reports', 'common']); + const isMobile = useIsMobile(); const [period, setPeriod] = useState('weekly'); const [report, setReport] = useState(null); const [loading, setLoading] = useState(true); @@ -120,6 +123,7 @@ export function ReportsPage({ authUser }: ReportsPageProps) { {/* Header with tabs */}
+ {isMobile && }

{t('title')}

diff --git a/packages/web-ui/src/pages/Search.tsx b/packages/web-ui/src/pages/Search.tsx new file mode 100644 index 00000000..814fe0a7 --- /dev/null +++ b/packages/web-ui/src/pages/Search.tsx @@ -0,0 +1,327 @@ +import { useState, useCallback, useRef, useEffect } from 'react'; +import { useTranslation } from 'react-i18next'; +import { api, type AgentInfo, type TaskInfo, type ProjectInfo, type DeliverableInfo, type RequirementInfo } from '../api.ts'; +import { navBus } from '../navBus.ts'; +import { PAGE } from '../routes.ts'; +import { Avatar } from '../components/Avatar.tsx'; + +type SearchCategory = 'all' | 'agents' | 'tasks' | 'requirements' | 'projects' | 'deliverables'; + +interface SearchResults { + agents: AgentInfo[]; + tasks: TaskInfo[]; + requirements: RequirementInfo[]; + projects: ProjectInfo[]; + deliverables: DeliverableInfo[]; +} + +export function SearchPage() { + const { t } = useTranslation(['common', 'home', 'work']); + const [query, setQuery] = useState(''); + const [category, setCategory] = useState('all'); + const [results, setResults] = useState({ agents: [], tasks: [], requirements: [], projects: [], deliverables: [] }); + const [loading, setLoading] = useState(false); + const [searched, setSearched] = useState(false); + const inputRef = useRef(null); + const debounceRef = useRef | undefined>(undefined); + + useEffect(() => { + setTimeout(() => inputRef.current?.focus(), 100); + }, []); + + const doSearch = useCallback(async (q: string) => { + if (!q.trim()) { + setResults({ agents: [], tasks: [], requirements: [], projects: [], deliverables: [] }); + setSearched(false); + return; + } + setLoading(true); + setSearched(true); + const lower = q.toLowerCase(); + try { + const [agentsRes, tasksRes, requirementsRes, projectsRes, deliverablesRes] = await Promise.allSettled([ + api.agents.list(), + api.tasks.list({ search: q, pageSize: 20 }), + api.requirements.list(), + api.projects.list(), + api.deliverables.search({ q, limit: 20 }), + ]); + + const agents = agentsRes.status === 'fulfilled' + ? agentsRes.value.agents.filter(a => a.name?.toLowerCase().includes(lower) || a.role?.toLowerCase().includes(lower)) + : []; + const tasks = tasksRes.status === 'fulfilled' ? tasksRes.value.tasks : []; + const requirements = requirementsRes.status === 'fulfilled' + ? requirementsRes.value.requirements.filter(r => r.title?.toLowerCase().includes(lower) || r.description?.toLowerCase().includes(lower)) + : []; + const projects = projectsRes.status === 'fulfilled' + ? projectsRes.value.projects.filter(p => p.name?.toLowerCase().includes(lower) || p.description?.toLowerCase().includes(lower)) + : []; + const deliverables = deliverablesRes.status === 'fulfilled' ? deliverablesRes.value.results : []; + + setResults({ agents, tasks, requirements, projects, deliverables }); + } catch { + setResults({ agents: [], tasks: [], requirements: [], projects: [], deliverables: [] }); + } finally { + setLoading(false); + } + }, []); + + const handleInput = (value: string) => { + setQuery(value); + if (debounceRef.current) clearTimeout(debounceRef.current); + debounceRef.current = setTimeout(() => doSearch(value), 400); + }; + + const categories: { id: SearchCategory; label: string }[] = [ + { id: 'all', label: t('common:all', { defaultValue: '全部' }) }, + { id: 'agents', label: t('common:agents', { defaultValue: '智能体' }) }, + { id: 'tasks', label: t('common:tasks', { defaultValue: '任务' }) }, + { id: 'requirements', label: t('common:requirements', { defaultValue: '需求' }) }, + { id: 'projects', label: t('common:projects', { defaultValue: '项目' }) }, + { id: 'deliverables', label: t('common:deliverables', { defaultValue: '交付物' }) }, + ]; + + const hasResults = results.agents.length > 0 || results.tasks.length > 0 || results.requirements.length > 0 || results.projects.length > 0 || results.deliverables.length > 0; + + return ( +
+ {/* Search header */} +
+
+ +
+ + handleInput(e.target.value)} + placeholder={t('common:search')} + className="w-full bg-surface-elevated border border-border-default rounded-xl pl-9 pr-3 py-2.5 text-sm text-fg-primary placeholder:text-fg-tertiary focus:border-brand-500 focus:outline-none transition-colors" + /> + {query && ( + + )} +
+
+
+ {categories.map(cat => ( + + ))} +
+
+ + {/* Results */} +
+ {loading && ( +
+
{t('common:loading')}
+
+ )} + + {!loading && searched && !hasResults && ( +
+ +

{t('common:noResults', { defaultValue: '没有找到相关结果' })}

+
+ )} + + {!loading && hasResults && ( +
+ {/* Agents */} + {(category === 'all' || category === 'agents') && results.agents.length > 0 && ( +
+

{t('common:agents', { defaultValue: '智能体' })}

+
+ {(category === 'all' ? results.agents.slice(0, 8) : results.agents).map(agent => ( + + ))} +
+ {category === 'all' && results.agents.length > 8 && ( + + )} +
+ )} + + {/* Tasks */} + {(category === 'all' || category === 'tasks') && results.tasks.length > 0 && ( +
+

{t('common:tasks', { defaultValue: '任务' })}

+
+ {(category === 'all' ? results.tasks.slice(0, 8) : results.tasks).map(task => ( + + ))} +
+ {category === 'all' && results.tasks.length > 8 && ( + + )} +
+ )} + + {/* Requirements */} + {(category === 'all' || category === 'requirements') && results.requirements.length > 0 && ( +
+

{t('common:requirements', { defaultValue: '需求' })}

+
+ {(category === 'all' ? results.requirements.slice(0, 8) : results.requirements).map(req => ( + + ))} +
+ {category === 'all' && results.requirements.length > 8 && ( + + )} +
+ )} + + {/* Projects */} + {(category === 'all' || category === 'projects') && results.projects.length > 0 && ( +
+

{t('common:projects', { defaultValue: '项目' })}

+
+ {(category === 'all' ? results.projects.slice(0, 8) : results.projects).map(proj => ( + + ))} +
+ {category === 'all' && results.projects.length > 8 && ( + + )} +
+ )} + + {/* Deliverables */} + {(category === 'all' || category === 'deliverables') && results.deliverables.length > 0 && ( +
+

{t('common:deliverables', { defaultValue: '交付物' })}

+
+ {(category === 'all' ? results.deliverables.slice(0, 8) : results.deliverables).map(d => ( + + ))} +
+ {category === 'all' && results.deliverables.length > 8 && ( + + )} +
+ )} +
+ )} + + {!loading && !searched && ( +
+ +

{t('common:search')}

+
+ )} +
+
+ ); +} + +function StatusDot({ status }: { status: string }) { + const color = status === 'active' || status === 'idle' ? 'bg-green-500' + : status === 'working' || status === 'busy' ? 'bg-blue-500' + : status === 'paused' ? 'bg-amber-500' + : 'bg-gray-400'; + return ; +} + +function TaskStatusIcon({ status }: { status: string }) { + const color = status === 'completed' ? 'text-green-500 bg-green-500/15' + : status === 'in_progress' ? 'text-blue-500 bg-blue-500/15' + : status === 'pending' ? 'text-amber-500 bg-amber-500/15' + : 'text-fg-tertiary bg-surface-elevated'; + return ( +
+ +
+ ); +} diff --git a/packages/web-ui/src/pages/Settings.tsx b/packages/web-ui/src/pages/Settings.tsx index f1b4dad6..53e1f51b 100644 --- a/packages/web-ui/src/pages/Settings.tsx +++ b/packages/web-ui/src/pages/Settings.tsx @@ -1,11 +1,12 @@ import { useEffect, useState, useCallback, useRef } from 'react'; import { useTranslation, Trans } from 'react-i18next'; -import { api, type StorageInfo, type OrphanInfo, type AuthUser, type HumanUserInfo } from '../api.ts'; +import { api, type StorageInfo, type OrphanInfo, type AuthUser, type HumanUserInfo, type RemoteStatus, hubApi, getHubUser, ensureHubAuth, wsClient } from '../api.ts'; import { THEME_OPTIONS, type ThemeMode } from '../hooks/useTheme.ts'; import { SUPPORTED_LANGUAGES } from '../i18n/index.ts'; import { navBus } from '../navBus.ts'; import { PAGE } from '../routes.ts'; import { Avatar, AvatarUpload } from '../components/Avatar.tsx'; +import { useIsMobile } from '../hooks/useIsMobile.ts'; interface ModelCost { input: number; output: number; cacheRead?: number; cacheWrite?: number } interface ModelDef { id: string; name: string; provider: string; contextWindow: number; maxOutputTokens: number; cost: ModelCost; reasoning?: boolean; inputTypes?: string[] } @@ -35,11 +36,61 @@ interface OllamaDetectResult { models?: Array<{ name: string; fullName: string; size?: number; modifiedAt?: string; parameterSize?: string; family?: string; quantization?: string }>; } +type SettingsTab = 'appearance' | 'providers' | 'execution' | 'browser' | 'search' | 'storage' | 'users' | 'remote'; + +const SETTINGS_TABS: Array<{ id: SettingsTab; labelKey: string; adminOnly?: boolean }> = [ + { id: 'appearance', labelKey: 'nav.appearance' }, + { id: 'providers', labelKey: 'nav.providers', adminOnly: true }, + { id: 'execution', labelKey: 'nav.execution', adminOnly: true }, + { id: 'browser', labelKey: 'nav.browser', adminOnly: true }, + { id: 'search', labelKey: 'nav.search', adminOnly: true }, + { id: 'storage', labelKey: 'nav.storage', adminOnly: true }, + { id: 'users', labelKey: 'nav.users', adminOnly: true }, + { id: 'remote', labelKey: 'nav.remote', adminOnly: true }, +]; + +function getSettingsTab(): SettingsTab | null { + const hash = window.location.hash.slice(1); + const parts = hash.split('/'); + if (parts[0] === 'settings' && parts[1]) { + const tab = parts[1] as SettingsTab; + if (SETTINGS_TABS.some(t => t.id === tab)) return tab; + } + return null; +} + export function Settings({ theme, onThemeChange, authUser, onLogout, onUserUpdated }: { theme?: ThemeMode; onThemeChange?: (m: ThemeMode) => void; authUser?: AuthUser; onLogout?: () => void; onUserUpdated?: (u: AuthUser) => void } = {}) { const { t, i18n } = useTranslation(['settings', 'common']); + const isMobile = useIsMobile(); const [userMenuOpen, setUserMenuOpen] = useState(false); const [showEditProfile, setShowEditProfile] = useState(false); const userMenuRef = useRef(null); + const [activeTab, setActiveTab] = useState(getSettingsTab); + + useEffect(() => { + const onHashChange = () => setActiveTab(getSettingsTab()); + window.addEventListener('hashchange', onHashChange); + return () => window.removeEventListener('hashchange', onHashChange); + }, []); + + const navigateTab = useCallback((tab: SettingsTab) => { + setActiveTab(tab); + history.pushState(null, '', `#settings/${tab}`); + }, []); + + const navigateBackToList = useCallback(() => { + setActiveTab(null); + history.pushState(null, '', '#settings'); + }, []); + + // On desktop, always show a tab (default to appearance). On mobile, null means show the list. + const resolvedTab: SettingsTab | null = activeTab ?? (isMobile ? null : 'appearance'); + + useEffect(() => { + const handler = () => setShowEditProfile(true); + window.addEventListener('markus:open-edit-profile', handler); + return () => window.removeEventListener('markus:open-edit-profile', handler); + }, []); useEffect(() => { if (!userMenuOpen) return; @@ -105,6 +156,9 @@ export function Settings({ theme, onThemeChange, authUser, onLogout, onUserUpdat const [browserBringToFront, setBrowserBringToFront] = useState(false); const [browserRemotePort, setBrowserRemotePort] = useState(0); const [browserAutoClose, setBrowserAutoClose] = useState(true); + const [browserAutoClickAllow, setBrowserAutoClickAllow] = useState(false); + const [browserExtensionConnected, setBrowserExtensionConnected] = useState(false); + const [browserExtensionPort, setBrowserExtensionPort] = useState(9333); const [browserSaving, setBrowserSaving] = useState(false); const [browserMsg, setBrowserMsg] = useState<{ type: 'ok' | 'err'; text: string } | null>(null); @@ -173,6 +227,9 @@ export function Settings({ theme, onThemeChange, authUser, onLogout, onUserUpdat setBrowserBringToFront(d.bringToFront ?? false); setBrowserRemotePort(d.remoteDebuggingPort ?? 0); setBrowserAutoClose(d.autoCloseTabs ?? true); + setBrowserAutoClickAllow(d.autoClickAllowDialog ?? false); + setBrowserExtensionConnected(d.extensionConnected ?? false); + setBrowserExtensionPort(d.extensionBridgePort ?? 9333); } }) .catch(() => {}); @@ -442,6 +499,21 @@ export function Settings({ theme, onThemeChange, authUser, onLogout, onUserUpdat if (oauthPollRef.current) clearInterval(oauthPollRef.current); }, []); + // Poll extension connection status when browser tab is active and not yet connected + useEffect(() => { + if (resolvedTab !== 'browser' || browserExtensionConnected) return; + const poll = setInterval(async () => { + try { + const d = await api.settings.getBrowser(); + if (d.extensionConnected) { + setBrowserExtensionConnected(true); + clearInterval(poll); + } + } catch { /* ignore */ } + }, 5000); + return () => clearInterval(poll); + }, [resolvedTab, browserExtensionConnected]); + // Add/Edit/Delete provider state const [showAddProvider, setShowAddProvider] = useState(false); const [addProviderForm, setAddProviderForm] = useState({ name: '', apiKey: '', baseUrl: '', model: '', contextWindow: 128000, maxOutputTokens: 16384, costInput: 1, costOutput: 5 }); @@ -677,8 +749,10 @@ export function Settings({ theme, onThemeChange, authUser, onLogout, onUserUpdat const showSetupGuide = llm && !hasConfiguredProviders && !setupDismissed; const canManageOrgSettings = authUser?.role === 'owner' || authUser?.role === 'admin'; + const visibleTabs = SETTINGS_TABS.filter(tab => !tab.adminOnly || canManageOrgSettings); + return ( -
+
{showEditProfile && authUser && ( )} -
-
-

{t('title')}

- {authUser && ( + {/* Settings Sidebar */} + + + {/* Content Panel */} +
+ {/* Mobile: settings list (when no sub-tab selected) */} + {isMobile && resolvedTab === null && ( +
+
+ +

{t('title')}

+
+
+ )} + {/* Mobile: sub-page header with back button */} + {isMobile && resolvedTab !== null && ( +
+ +

{t(`settings:${visibleTabs.find(tb => tb.id === resolvedTab)?.labelKey || 'title'}`)}

+
+ )} + {resolvedTab !== null &&
{/* ───── Appearance ───── */} -
+ {resolvedTab === 'appearance' &&
@@ -772,11 +916,12 @@ export function Settings({ theme, onThemeChange, authUser, onLogout, onUserUpdat
-
+
} {canManageOrgSettings && ( <> - {/* ───── First-Run Setup Guide ───── */} + {/* ───── First-Run Setup Guide (shown in providers tab) ───── */} + {resolvedTab === 'providers' && <> {showSetupGuide && (
+ +
+
{t('setupGuide.browserExtension.loadHint')}
+
+ )} + + )}
)} @@ -910,6 +1100,7 @@ export function Settings({ theme, onThemeChange, authUser, onLogout, onUserUpdat {/* ───── Default Provider ───── */} +
@@ -1511,7 +1702,9 @@ export function Settings({ theme, onThemeChange, authUser, onLogout, onUserUpdat
+ } + {resolvedTab === 'execution' && <>
@@ -1607,20 +1800,42 @@ export function Settings({ theme, onThemeChange, authUser, onLogout, onUserUpdat {cppMsg && }
+ } + {resolvedTab === 'browser' && <>
-
-
- {t('browserAutomation.description')} + {/* ── Active Mode Banner ── */} +
0 ? 'bg-blue-500/10 border border-blue-500/20' : browserAutoClickAllow ? 'bg-amber-500/10 border border-amber-500/20' : 'bg-surface-elevated border border-border-default'}`}> +
0 ? 'bg-blue-400' : browserAutoClickAllow ? 'bg-amber-400' : 'bg-gray-400'}`} /> +
+
0 ? 'text-blue-400' : browserAutoClickAllow ? 'text-amber-400' : 'text-fg-secondary'}`}> + {browserExtensionConnected + ? t('browserAutomation.modeExtension') + : browserRemotePort > 0 + ? t('browserAutomation.modeDebuggingPort', { port: browserRemotePort }) + : browserAutoClickAllow + ? t('browserAutomation.modeAutoClick') + : t('browserAutomation.modeManual')} +
+
+ {browserExtensionConnected + ? t('browserAutomation.modeExtensionDesc') + : browserRemotePort > 0 + ? t('browserAutomation.modeDebuggingPortDesc') + : browserAutoClickAllow + ? t('browserAutomation.modeAutoClickDesc') + : t('browserAutomation.modeManualDesc')} +
+
- {/* Bring to Front toggle */} + {/* ── General Settings (always visible) ── */} +
+
{t('browserAutomation.generalSettings')}
{t('browserAutomation.bringToFront')}
-
- {t('browserAutomation.bringToFrontDesc')} -
+
{t('browserAutomation.bringToFrontDesc')}
- - {/* Auto-close tabs toggle */}
{t('browserAutomation.autoCloseTabs')}
-
- {t('browserAutomation.autoCloseTabsDesc')} -
+
{t('browserAutomation.autoCloseTabsDesc')}
+
- {/* Remote Debugging Port */} -
-
-
-
{t('browserAutomation.remoteDebuggingPort')}
-
- {t('browserAutomation.remoteDebuggingPortDescLead')}{' '} - --remote-debugging-port=9222{' '} - {t('browserAutomation.remoteDebuggingPortDescTail')} + {/* ── Connection Mode: Chrome Extension ── */} +
+ {/* Header row: title + status badge */} +
+
+
{t('browserAutomation.extensionStatus')}
+ ({t('browserAutomation.modeRecommended')}) +
+ + + {browserExtensionConnected ? t('browserAutomation.extensionConnected') : t('browserAutomation.extensionDisconnected')} + +
+
{t('browserAutomation.extensionStatusDesc')}
+ + {browserExtensionConnected ? ( +
+ + {t('browserAutomation.extensionActiveNote')} +
+ ) : ( + /* Step-by-step install guide */ +
+
{t('browserAutomation.extensionSetupTitle')}
+ + {/* Step 1: Download */} +
+ 1 +
+
{t('browserAutomation.extensionStep1')}
+
{t('browserAutomation.extensionStep1Desc')}
+
-
- { setBrowserRemotePort(Number(e.target.value)); setBrowserMsg(null); }} - className="w-24 px-3 py-1.5 text-sm border border-border-default rounded-lg bg-surface-primary text-fg-primary text-right" - placeholder="0" - /> - + + {/* Step 2: Open extensions page */} +
+ 2 +
+
{t('browserAutomation.extensionStep2')}
+
{t('browserAutomation.extensionStep2Desc')}
+ +
+
+ + {/* Step 3: Load unpacked */} +
+ 3 +
+
{t('browserAutomation.extensionStep3')}
+
{t('browserAutomation.extensionStep3Desc')}
+
+
+
+ )} +
+ + {/* ── Fallback modes (collapsed when extension is connected) ── */} + {!browserExtensionConnected && ( +
+
{t('browserAutomation.fallbackModes')}
+ + {/* Auto-click Chrome Allow Dialog toggle */} +
+
+
+
{t('browserAutomation.autoClickAllowDialog')}
+
{t('browserAutomation.autoClickAllowDialogDesc')}
+
+
{t('browserAutomation.autoClickAllowDialogMacNote')}
+
{t('browserAutomation.autoClickAllowDialogWinNote')}
+
{t('browserAutomation.autoClickAllowDialogLinuxNote')}
+
+
+
+ + +
+
+
+ + {/* Remote Debugging Port */} +
+
+
+
{t('browserAutomation.remoteDebuggingPort')}
+
+ {t('browserAutomation.remoteDebuggingPortDescLead')}{' '} + --remote-debugging-port=9222{' '} + {t('browserAutomation.remoteDebuggingPortDescTail')} +
+
+
+ { setBrowserRemotePort(Number(e.target.value)); setBrowserMsg(null); }} + className="w-24 px-3 py-1.5 text-sm border border-border-default rounded-lg bg-surface-primary text-fg-primary text-right" + placeholder="0" + /> + +
+ )} - {browserMsg && } -
+ {browserMsg &&
}
+ } + {resolvedTab === 'search' && <>
{t('searchApi.description')}
@@ -1773,7 +2131,9 @@ export function Settings({ theme, onThemeChange, authUser, onLogout, onUserUpdat
+ } + {resolvedTab === 'storage' && <>
{storageLoading && !storageInfo &&
{t('dataStorage.scanning')}
} @@ -1864,12 +2224,17 @@ export function Settings({ theme, onThemeChange, authUser, onLogout, onUserUpdat
- + } + + {resolvedTab === 'users' && } + + {resolvedTab === 'remote' && } )}
+
}
); @@ -2287,3 +2652,284 @@ function EditProfileModal({ authUser, onClose, onSaved }: { authUser: AuthUser;
); } + +/* ─── Remote Access ─── */ + +function RemoteAccessSection() { + const { t } = useTranslation(['settings', 'common']); + const [status, setStatus] = useState(null); + const [loading, setLoading] = useState(true); + const [toggling, setToggling] = useState(false); + const [error, setError] = useState(null); + const [copied, setCopied] = useState(false); + const hubUser = getHubUser(); + + const loadStatus = useCallback(async () => { + try { + const s = await api.settings.getRemote(); + setStatus(s); + } catch { + setStatus(null); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { loadStatus(); }, [loadStatus]); + + useEffect(() => { + return wsClient.on('remote:status', (event) => { + const payload = event.payload as unknown as RemoteStatus | undefined; + if (payload) { + setStatus(payload); + setToggling(false); + } + }); + }, []); + + const handleToggle = async () => { + setToggling(true); + setError(null); + try { + if (status?.enabled) { + await api.settings.disableRemote(); + await loadStatus(); + setToggling(false); + } else { + if (!hubApi.isAuthenticated()) { + await ensureHubAuth(); + } + await api.settings.enableRemote(); + } + } catch (err) { + setError(String(err instanceof Error ? err.message : err)); + setToggling(false); + } + }; + + const handleLogin = async () => { + try { + await ensureHubAuth(); + await loadStatus(); + } catch { /* user cancelled */ } + }; + + const handleCopy = () => { + if (!status?.remoteUrl) return; + navigator.clipboard.writeText(status.remoteUrl); + setCopied(true); + setTimeout(() => setCopied(false), 2000); + }; + + const isConnecting = toggling || (status?.enabled && !status?.connected && status?.state !== 'idle'); + const qrUrl = status?.remoteUrl ?? null; + + return ( +
+
+

+ {t('settings:remoteAccess.description')} +

+ + {/* Hub Auth Status */} + {!hubApi.isAuthenticated() ? ( +
+ + + + + {t('settings:remoteAccess.loginRequired')} + + +
+ ) : ( +
+ + {t('settings:remoteAccess.signedInAs')} {hubUser?.username ?? hubUser?.displayName} +
+ )} + + {/* Toggle + Status */} + {hubApi.isAuthenticated() && ( +
+
+
+ + + {isConnecting + ? t('settings:remoteAccess.connecting') + : status?.enabled + ? t('settings:remoteAccess.enabled') + : t('settings:remoteAccess.disabled')} + + {isConnecting && } +
+ + {status?.enabled && ( +
+ + {status.connected + ? t('settings:remoteAccess.connected') + : (status.state === 'registering') + ? t('settings:remoteAccess.registering') + : (status.state === 'connecting') + ? t('settings:remoteAccess.connecting') + : t('settings:remoteAccess.disconnected')} + {status.connected && status.peerCount > 0 && ( + <> · {t('settings:remoteAccess.peerCount', { count: status.peerCount })} + )} + {(status.state === 'registering' || status.state === 'connecting') && } +
+ )} +
+ + {error && ( +
{error}
+ )} + + {/* Connected Peers List */} + {status?.enabled && status.peers && status.peers.length > 0 && ( +
+ +
+ {status.peers.map((peer) => ( +
+ +
+
+ {peer.peerId.slice(0, 8)}... +
+
+ {t('settings:remoteAccess.connectedSince', { time: new Date(peer.connectedAt).toLocaleTimeString() })} +
+
+ + {peer.transport === 'p2p' ? 'P2P' : peer.transport === 'relay' ? 'Relay' : t('settings:remoteAccess.peerConnecting')} + +
+ ))} +
+
+ )} + + {/* Remote URL + QR Code */} + {status?.enabled && status.remoteUrl && ( +
+
+ +
+ + {status.remoteUrl} + + +
+
+ + {qrUrl && ( +
+ +

+ {t('settings:remoteAccess.scanQr')} +

+ +
+ )} +
+ )} +
+ )} +
+
+ ); +} + +function Spinner() { + return ( + + + + + ); +} + +function QRCode({ url }: { url: string }) { + const canvasRef = useRef(null); + const [error, setError] = useState(false); + + useEffect(() => { + const canvas = canvasRef.current; + if (!canvas) return; + setError(false); + + import('qrcode').then((QRLib) => { + QRLib.toCanvas(canvas, url, { + width: 200, + margin: 2, + color: { dark: '#000000', light: '#ffffff' }, + errorCorrectionLevel: 'M', + }); + }).catch(() => setError(true)); + }, [url]); + + if (error) { + return ( + {url} + ); + } + + return ( +
+ +
+ ); +} diff --git a/packages/web-ui/src/pages/Team.tsx b/packages/web-ui/src/pages/Team.tsx index f4a47a48..04b674d2 100644 --- a/packages/web-ui/src/pages/Team.tsx +++ b/packages/web-ui/src/pages/Team.tsx @@ -29,6 +29,7 @@ import { TeamProfile, TABS as TEAM_TABS, type TeamTab } from './TeamProfile.tsx' import { useResizablePanel } from '../hooks/useResizablePanel.ts'; import { useIsMobile } from '../hooks/useIsMobile.ts'; import { useSwipeTabs } from '../hooks/useSwipeTabs.ts'; +import { useUnreadCounts } from '../hooks/useUnreadCounts.ts'; import { Avatar } from '../components/Avatar.tsx'; // ─── Helpers ────────────────────────────────────────────────────────────────── @@ -798,14 +799,29 @@ export function TeamPage({ initialAgentId, authUser }: { initialAgentId?: string const [initialLoading, setInitialLoading] = useState(true); const isMobile = useIsMobile(); - // Mobile: URL hash is the single source of truth (#chat = list, #chat/d = detail) + // Mobile: URL hash is the single source of truth for 3-layer navigation + // L1 (roster): #team — sidebar list + // L2 (team detail): #team/t/ — team agent list + channel + // L3 (chat): #team/d — agent/channel chat const hash = useSyncExternalStore(_subHash, _getHash); const mobileShowChat = isMobile && (hash.startsWith(`#${PAGE.TEAM}/`) || hash.startsWith('#chat/')); - + const mobileTeamHash = isMobile && hash.match(/^#team\/t\/(.+)$/); + const mobileLayer: 'roster' | 'team' | 'chat' = !isMobile ? 'roster' + : mobileTeamHash ? 'team' + : mobileShowChat ? 'chat' + : 'roster'; + const mobileTeamId = mobileTeamHash ? mobileTeamHash[1] : null; + + const mobileBackHashRef = useRef(PAGE.TEAM); const enterMobileDetail = useCallback(() => { + mobileBackHashRef.current = window.location.hash.slice(1) || PAGE.TEAM; window.location.hash = `${PAGE.TEAM}/d`; }, []); + const enterMobileTeam = useCallback((teamId: string) => { + window.location.hash = `${PAGE.TEAM}/t/${teamId}`; + }, []); + // Profile tab: still uses pushState for back navigation useEffect(() => { if (!isMobile) return; @@ -1032,6 +1048,9 @@ export function TeamPage({ initialAgentId, authUser }: { initialAgentId?: string // Group chats const [groupChats, setGroupChats] = useState }>>([]); + const groupChatsRef = useRef(groupChats); + groupChatsRef.current = groupChats; + const pendingSelectTeamRef = useRef(null); const [showMemberPanel, setShowMemberPanel] = useState(false); // Teams @@ -1135,14 +1154,18 @@ export function TeamPage({ initialAgentId, authUser }: { initialAgentId?: string // ── Per-agent unread notification counts + auto-read ──────────────────────── const [unreadByAgent, setUnreadByAgent] = useState>(new Map()); const unreadIdsByAgent = useRef>(new Map()); + const agentIdsRef = useRef>(new Set()); + useEffect(() => { agentIdsRef.current = new Set(agents.map(a => a.id)); }, [agents]); + const refreshUnreadCounts = useCallback(async () => { try { const { notifications } = await api.notifications.list(authUser?.id, true); const counts = new Map(); const ids = new Map(); + const knownAgents = agentIdsRef.current; for (const n of notifications) { const agentId = (n.metadata?.agentId as string) || undefined; - if (agentId) { + if (agentId && knownAgents.has(agentId)) { counts.set(agentId, (counts.get(agentId) ?? 0) + 1); const list = ids.get(agentId) ?? []; list.push(n.id); @@ -1151,6 +1174,10 @@ export function TeamPage({ initialAgentId, authUser }: { initialAgentId?: string } setUnreadByAgent(counts); unreadIdsByAgent.current = ids; + // Broadcast total for BottomNav badge + let total = 0; + for (const v of counts.values()) total += v; + window.dispatchEvent(new CustomEvent('markus:team-unread-changed', { detail: { count: total } })); } catch { /* */ } }, [authUser?.id]); @@ -1173,6 +1200,33 @@ export function TeamPage({ initialAgentId, authUser }: { initialAgentId?: string } }, [chatMode, selectedAgent, markAgentNotificationsRead]); + // ── Chat unread counts (message-level read cursors) ────────────────────────── + const { counts: chatUnreadCounts, markRead: markChatRead } = useUnreadCounts(); + const unreadByChannel = useMemo(() => { + const result: Record = {}; + for (const [key, count] of Object.entries(chatUnreadCounts)) { + if (key.startsWith('channel:')) { + result[key.slice('channel:'.length)] = count; + } + } + return result; + }, [chatUnreadCounts]); + + // Auto mark-read when a conversation becomes visible + useEffect(() => { + const isVisible = !isMobile || mobileLayer === 'chat'; + if (!isVisible) return; + if (chatMode === 'channel' && activeChannel) { + markChatRead(`channel:${activeChannel}`); + } else if (chatMode === 'direct' && activeSessionId) { + markChatRead(`session:${activeSessionId}`); + } else if (chatMode === 'dm' && activeDmUserId) { + const dmChannel = `dm:${[authUser?.id, activeDmUserId].sort().join(':')}`; + markChatRead(`channel:${dmChannel}`); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [chatMode, activeChannel, activeSessionId, activeDmUserId, mobileLayer]); + // ── Data loading ───────────────────────────────────────────────────────────── const refreshAgents = useCallback(() => api.agents.list().then(d => setAgents(d.agents)).catch(() => {}), []); const refreshTeams = useCallback(() => api.teams.list().then(d => setTeams(d.teams)).catch(() => {}), []); @@ -1224,6 +1278,8 @@ export function TeamPage({ initialAgentId, authUser }: { initialAgentId?: string } else { setChatMode('direct'); setSelectedAgent(detail.params.agentId); + setMainTab('chat'); + if (isMobile) enterMobileDetail(); if (detail.params.sessionId) { const targetSessionId = detail.params.sessionId; setTimeout(async () => { @@ -1247,11 +1303,22 @@ export function TeamPage({ initialAgentId, authUser }: { initialAgentId?: string setChatMode('dm'); setActiveDmUserId(detail.params.dm); setMainTab('chat'); + if (isMobile) enterMobileDetail(); } if (detail.params?.channel) { setChatMode('channel'); setActiveChannel(detail.params.channel); setMainTab('chat'); + if (isMobile) enterMobileDetail(); + } + if (detail.params?.selectTeam) { + const teamId = detail.params.selectTeam; + if (isMobile) { + enterMobileTeam(teamId); + } else { + const teamGc = groupChatsRef.current.find(gc => gc.type === 'team' && gc.teamId === teamId); + if (teamGc) { setChatMode('channel'); setActiveChannel(teamGc.channelKey); setMainTab('chat'); setShowMemberPanel(false); setShowTeamDetailPanel(true); } + } } if (detail.params?.openHire === 'true') { // handled by ChatTeamSidebar via nav events @@ -1269,22 +1336,43 @@ export function TeamPage({ initialAgentId, authUser }: { initialAgentId?: string if (navDm) { localStorage.removeItem('markus_nav_dm'); setChatMode('dm'); setActiveDmUserId(navDm); setMainTab('chat'); + if (isMobile) enterMobileDetail(); } const navChannel = localStorage.getItem('markus_nav_channel'); if (navChannel) { localStorage.removeItem('markus_nav_channel'); setChatMode('channel'); setActiveChannel(navChannel); setMainTab('chat'); + if (isMobile) enterMobileDetail(); } const selectAgent = localStorage.getItem('markus_nav_selectAgent'); if (selectAgent) { localStorage.removeItem('markus_nav_selectAgent'); handleViewProfile(selectAgent); } + const selectTeam = localStorage.getItem('markus_nav_selectTeam'); + if (selectTeam) { + localStorage.removeItem('markus_nav_selectTeam'); + if (isMobile) { + enterMobileTeam(selectTeam); + } else { + pendingSelectTeamRef.current = selectTeam; + } + } window.addEventListener('markus:navigate', handleNav); return () => window.removeEventListener('markus:navigate', handleNav); // eslint-disable-next-line react-hooks/exhaustive-deps }, []); + useEffect(() => { + const teamId = pendingSelectTeamRef.current; + if (!teamId || groupChats.length === 0) return; + const teamGc = groupChats.find(gc => gc.type === 'team' && gc.teamId === teamId); + if (teamGc) { + pendingSelectTeamRef.current = null; + setChatMode('channel'); setActiveChannel(teamGc.channelKey); setMainTab('chat'); setShowMemberPanel(false); setShowTeamDetailPanel(true); + } + }, [groupChats]); + // Auto-select secretary agent when no valid agent is selected. // Also handles stale IDs from localStorage (e.g. deleted agents). useEffect(() => { @@ -2610,7 +2698,7 @@ export function TeamPage({ initialAgentId, authUser }: { initialAgentId?: string selectedAgent ? t('page.placeholder.direct') : t('page.placeholder.noAgent'); // ── Render ──────────────────────────────────────────────────────────────────── - const showChatOnMobile = isMobile && mobileShowChat; + const showChatOnMobile = isMobile && mobileLayer === 'chat'; return (
@@ -2633,7 +2721,7 @@ export function TeamPage({ initialAgentId, authUser }: { initialAgentId?: string onSelectTeam={(teamId) => { const teamGc = groupChats.find(gc => gc.type === 'team' && gc.teamId === teamId); if (isMobile) { - if (teamGc) { setChatMode('channel'); setActiveChannel(teamGc.channelKey); setMainTab('chat'); setShowMemberPanel(false); enterMobileDetail(); } + enterMobileTeam(teamId); } else { if (teamGc) { setChatMode('channel'); setActiveChannel(teamGc.channelKey); setMainTab('chat'); setShowMemberPanel(false); if (!showTeamDetailPanel && !l2SpaceTight) setShowTeamDetailPanel(true); } } @@ -2646,12 +2734,92 @@ export function TeamPage({ initialAgentId, authUser }: { initialAgentId?: string onViewProfile={handleViewProfile} onManageGroupMembers={(channelKey) => { setChatMode('channel'); setActiveChannel(channelKey); setMainTab('chat'); setShowMemberPanel(true); if (isMobile) enterMobileDetail(); }} unreadByAgent={unreadByAgent} + unreadByChannel={unreadByChannel} width={isMobile ? undefined : chatSidebar.width} onResizeStart={isMobile ? undefined : chatSidebar.onResizeStart} - hidden={isMobile && mobileShowChat} + hidden={isMobile && mobileLayer !== 'roster'} initialLoading={initialLoading} /> + {/* ── L2: Mobile team detail view ── */} + {isMobile && mobileLayer === 'team' && mobileTeamId && (() => { + const l2Team = teams.find(t => t.id === mobileTeamId); + if (!l2Team) return null; + const l2Agents = agents.filter(a => a.teamId === mobileTeamId); + const l2Gc = groupChats.find(gc => gc.type === 'team' && gc.teamId === mobileTeamId); + return ( +
+
+ +
+
+ +
+
+

{l2Team.name}

+

{t('chat.members_other', { count: l2Team.members?.length || l2Agents.length })}

+
+
+
+
+ {l2Gc && (() => { + const gcUnread = unreadByChannel[l2Gc.channelKey] ?? 0; + return ( + + ); + })()} + {l2Agents.length > 0 && ( + <> +

{t('chat.agents')}

+ {l2Agents.map(agent => { + const agentUnread = unreadByAgent.get(agent.id) ?? 0; + return ( + + ); + })} + + )} +
+
+ ); + })()} + {/* ── L2: Team detail panel (desktop only) ── */} {/* Inline mode: when space allows */} {showTeamDetailPanel && !l2SpaceTight && !isMobile && (() => { @@ -2730,7 +2898,7 @@ export function TeamPage({ initialAgentId, authUser }: { initialAgentId?: string {/* Mobile Row 1: back + name + status */}