From 1337e8b65b6f752526d2429bf3b94df01798f59b Mon Sep 17 00:00:00 2001 From: wolframs91 Date: Wed, 8 Jul 2026 21:26:48 +0200 Subject: [PATCH 1/3] feat(relay,mcpl): markdown-preserving split sends + opt-in inline audio (RFC-006) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports chapterx PR #13 (markdown across message splits) and PR #12 (audio input; membrane PR #29 context) onto portal's relay architecture. Split sends (relay): - discord-markdown.ts: splitter engine ported from chapterx β€” closes/reopens fences & emphasis at chunk boundaries, records synthetic bridge strings - sendMessage splits >2000-char content; WebhookPool.sendMany keeps parts contiguous on the persona's webhook queue (send() now delegates to it); partial failures record the posted parts and name them in the error - MessageStore persists bridgeOpen/bridgeClose/parts/partOf per part; buildPortalMessage strips bridges so agents see their original markdown (history, live events, pins), with PortalMessage.partOf for grouping - edit_message re-splits across existing parts (surplus deleted, growth refused with INVALID_PARAMS); delete_message fans out over all parts and surfaces failures instead of silently succeeding - SendMessageResult.messageIds lists all parts (additive, no version bump) Inline audio (mcpl + protocol): - PortalAttachment.duration/waveform captured from Discord voice messages - set_audio_visibility tool: durable per-channel opt-in (default OFF), mirroring the reactions pattern; content-shaping only, never wakes - buildContent inlines audio as MCPL audio blocks (12MB cap, max 2/message, MIME normalized: mpeg/mpeg3/x-mpeg-3 β†’ mp3, extension fallback when Discord omits contentType); attachment fetches now run concurrently - oversized clips flagged with a native 🐘 reaction (idempotent shared-bot) Tests: 140 splitter tests ported from chapterx + new split-send integration tests (contiguity, partial failure, store round-trip, strip/reassemble) and audio tests (opt-in persistence, MIME normalization/detection). 257 total. Co-Authored-By: Claude Fable 5 --- ...AL-RFC-006-split-sends-and-inline-audio.md | 94 ++ README.md | 10 +- portal-client/package-lock.json | 12 +- portal-mcpl/package-lock.json | 18 +- portal-mcpl/src/agent-state.ts | 28 + portal-mcpl/src/agent.ts | 10 + portal-mcpl/src/server.ts | 124 +- portal-mcpl/src/tools.ts | 17 + portal-mcpl/test/audio.test.ts | 46 + portal-protocol/package-lock.json | 4 +- portal-protocol/src/message.ts | 8 + portal-protocol/src/rpc.ts | 5 + portal-relay/package-lock.json | 6 +- portal-relay/src/discord-bot.ts | 6 + portal-relay/src/discord-markdown.ts | 769 ++++++++++ portal-relay/src/message-store.ts | 38 + portal-relay/src/relay.ts | 213 ++- portal-relay/src/webhook-pool.ts | 51 +- portal-relay/test/discord-markdown.test.ts | 1360 +++++++++++++++++ portal-relay/test/split-send.test.ts | 149 ++ 20 files changed, 2900 insertions(+), 68 deletions(-) create mode 100644 PORTAL-RFC-006-split-sends-and-inline-audio.md create mode 100644 portal-mcpl/test/audio.test.ts create mode 100644 portal-relay/src/discord-markdown.ts create mode 100644 portal-relay/test/discord-markdown.test.ts create mode 100644 portal-relay/test/split-send.test.ts diff --git a/PORTAL-RFC-006-split-sends-and-inline-audio.md b/PORTAL-RFC-006-split-sends-and-inline-audio.md new file mode 100644 index 0000000..42376b6 --- /dev/null +++ b/PORTAL-RFC-006-split-sends-and-inline-audio.md @@ -0,0 +1,94 @@ +# PORTAL RFC-006 β€” Markdown-preserving split sends + inline audio + +- **Status:** βœ… Implemented (2026-07-08) +- **Author:** Wolfram (drafted with Claude Code) +- **Date:** 2026-07-08 +- **Affects:** `portal-protocol` (`rpc.ts` `SendMessageResult`, `message.ts` `PortalMessage.partOf` + `PortalAttachment.duration/waveform`), `portal-relay` (`discord-markdown.ts` NEW, `relay.ts`, `webhook-pool.ts`, `message-store.ts`, `discord-bot.ts`), `portal-mcpl` (`server.ts`, `agent-state.ts`, `tools.ts`, `agent.ts`) +- **Protocol version:** additive only β€” no `PORTAL_PROTOCOL_VERSION` bump (older clients never see the new optional fields). +- **Provenance:** ports chapterx `fix/markdown-across-message-splits` (PR #13) and `feat/audio-input` (PR #12, with membrane PR #29 context) onto portal's relay architecture. + +--- + +## 1. Markdown-preserving split sends (relay) + +**Problem.** The relay had no handling for Discord's 2000-char content limit: an +over-long `send_message` bubbled a raw discord.js error up as `INTERNAL`. And a +naive split would corrupt markdown β€” Discord parses each message independently, +so a code fence or emphasis straddling a boundary breaks everything after it. + +**Design.** + +- `portal-relay/src/discord-markdown.ts` β€” the splitter engine, ported verbatim + (modulo formatting) from chapterx. `splitPreservingMarkdown(text, 2000)` + closes every construct open at a chunk boundary and reopens it in the next + chunk, records the synthetic `bridgeOpen`/`bridgeClose` strings per chunk, + caps fence info-strings so a reopener can't blow the budget, and avoids + fusing synthetic delimiters with adjacent runs. Keep in sync upstream. +- **Send** (`relay.ts sendMessage`): content ≀ 2000 β†’ unchanged single-send + path. Longer β†’ split; parts go out via `WebhookPool.sendMany`, which enqueues + ALL parts as one item on the persona's webhook queue so no same-webhook send + can interleave between them (single sends delegate to the same path). Files + ride on the last part. A mid-sequence failure records the parts that DID land + (attributable/editable) and fails the RPC with `DISCORD_ERROR` naming their + relay ids, so the caller can delete them or send the remainder. +- **Result shape**: `SendMessageResult.messageIds?` lists every part (first = + `messageId`). `PortalMessage.partOf?` marks continuation parts so clients can + group them. +- **Store** (`message-store.ts`): refs/attribution rows carry + `bridgeOpen`/`bridgeClose`/`parts` (first part)/`partOf` (continuations) β€” + persisted, so bridge stripping and whole-set edit/delete survive restarts. +- **Bridge stripping** (`relay.ts buildPortalMessage` β†’ `stripBridges`): the one + choke point where Discord messages become `PortalMessage`s (live events, + fetch_history, pins) removes the synthetic markers, so agents always see + their original unbroken markdown. A fence reopener whose info string Discord + rewrote (mentions in `cleanContent`) is matched on the fence marker run alone. +- **Edit** (`edit_message`): redirected to the first part; new content is + re-split and written across the existing parts; surplus parts are deleted; + bridge metadata is rewritten (`setSplitMeta`). Needing MORE parts than the + original send is refused with `INVALID_PARAMS` (Discord can't insert messages + in place) β€” shorten, or delete and resend. +- **Delete** (`delete_message`): fans out over all parts (both own-webhook and + moderation paths). Every part is attempted; any failure still surfaces as + `DISCORD_ERROR` (never a silent success). A surplus part whose delete fails + during an edit keeps its store ref so it stays addressable. + +**Verified:** `portal-relay/test/discord-markdown.test.ts` (140 tests, ported +from chapterx) and `test/split-send.test.ts` (sendMany ordering/contiguity/ +partial failure, store round-trip, stripBridges, splitβ†’strip reassembly). + +## 2. Inline audio for agents (mcpl) + protocol audio metadata + +**Problem.** Audio attachments (voice messages, music files) reached agents only +as URL text notes β€” an audio-capable model behind portal could never hear them. + +**Design.** + +- **Protocol**: `PortalAttachment` gains optional `duration` (seconds) and + `waveform` (base64 preview) β€” Discord voice-message fields the relay now + captures from discord.js (`discord-bot.ts`). +- **Opt-in, per channel, default OFF** (mirrors the RFC-A reactions pattern): + durable `audioChannels` set on `AgentState`, toggled by the new + `set_audio_visibility` tool. Content-shaping only β€” wake behavior unchanged + (`chat:has-audio` tagging was already there; it now also catches + extension-detected audio with no content-type). +- **Inlining** (`server.ts buildContent`): when the channel is opted in, audio + attachments are fetched (15s timeout) and delivered as MCPL + `{type:'audio', data, mimeType}` blocks (`mcpl-core` already declares + `AudioContent`). Detection: `audio/*` content-type OR filename extension + (Discord's `contentType` is optional). MIME is normalized (MP3 aliases + `audio/mpeg`, `audio/mpeg3`, `audio/x-mpeg-3` β†’ `audio/mp3`; parameters + stripped) so provider layers (e.g. membrane β‰₯ PR #29 behind connectome) can + map it. Caps: 12 MiB raw per clip (~16 MiB base64), max 2 clips per message; + over-cap or failed fetches degrade to a text note (now with duration). +- **Oversized flag**: an opted-in agent seeing a >12 MiB clip reacts natively + 🐘 via the existing `react` RPC (shared-bot reaction β€” idempotent across + multiple agents), so the sender knows nobody can hear it. Ported from + chapterx's `oversized_audio_emote`. +- **Non-goals**: no relay-side audio fetching/caching (the relay stays a thin + id/attribution layer; each opted-in agent fetches from the Discord CDN + directly), no transcription, no voice-channel support, no Claude Code channel + inlining (`server-cc.ts` payloads are plain strings β€” attachments stay URL + lines there). + +**Verified:** `portal-mcpl/test/audio.test.ts` (opt-in toggle + serialization, +MIME normalization, detection fallback chain). diff --git a/README.md b/README.md index 5062cce..f841079 100644 --- a/README.md +++ b/README.md @@ -138,10 +138,11 @@ Wiring examples: `portal-mcpl/examples/mcpl-servers.json`, | Capability | Outbound (agentβ†’Discord) | Inbound (Discordβ†’agent) | |---|---|---| -| Text messages | βœ… | βœ… | +| Text messages | βœ… (over-long sends auto-split, markdown-preserving β€” RFC-006) | βœ… (split parts served with original markdown restored) | | Replies | βœ… (quoted jump-link; webhooks lack native reply) | βœ… `replyToId` resolved | | Threads | βœ… (parent webhook + threadId) | βœ… as channels w/ parentId | | Images / files | βœ… inline base64 `bytes` (RFC-003; path-files default-off) | βœ… inlined as MCPL image blocks (≀5 MB) + notes for non-images | +| Audio / voice messages | βœ… generic file `bytes` | βœ… opt-in inline audio blocks (≀12 MB, per-channel `set_audio_visibility` β€” RFC-006) + `duration`/`waveform` metadata | | Mentions | βœ… personaβ†’role; human `@name`β†’`<@id>` | βœ… roleβ†’persona routing + full `mentions` | | Reactions | βœ… pseudo + visible | βœ… native `reaction_add`/`reaction_remove` | | Typing | βœ… bot-level (anonymous β€” not per-persona) | n/a | @@ -173,6 +174,13 @@ greenfield agents: See `PORTAL-RFC-00{1,2,3}-*.md`. +- **RFC-006 β€” split sends + inline audio.** Over-long sends are split + markdown-preservingly (fences/emphasis closed + reopened per part; agents see + their original markdown restored via bridge stripping; edit/delete span all + parts). Audio attachments reach opted-in agents as playable MCPL audio blocks + with voice-message `duration`/`waveform` on the wire. See + `PORTAL-RFC-006-split-sends-and-inline-audio.md`. + ## Design decisions - **No DMs** β€” webhooks can't post to DMs. A later web surface + a `/dm-link` diff --git a/portal-client/package-lock.json b/portal-client/package-lock.json index dac9674..5509ab2 100644 --- a/portal-client/package-lock.json +++ b/portal-client/package-lock.json @@ -1,14 +1,14 @@ { "name": "@animalabs/portal-client", - "version": "0.2.0", + "version": "0.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@animalabs/portal-client", - "version": "0.2.0", + "version": "0.3.0", "dependencies": { - "@animalabs/portal-protocol": "^0.2.0", + "@animalabs/portal-protocol": "^0.3.0", "ws": "^8.18.0" }, "devDependencies": { @@ -35,9 +35,9 @@ } }, "node_modules/@animalabs/portal-protocol": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/@animalabs/portal-protocol/-/portal-protocol-0.2.0.tgz", - "integrity": "sha512-TbuwvdUmDwWyp3oxojSM81O+l8F3IbjjAcGLKmcgx1CY3/8x0j1ypNYaK0iUQ9oinJQuRPARsVom0s4k4G9T7g==", + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@animalabs/portal-protocol/-/portal-protocol-0.3.0.tgz", + "integrity": "sha512-tu1eqNIEtTUxrojUlcAEczWh9aFXH1qLXWGu7232Bi3r+Nyz+Y1lUq5sGzr14JcljstFtiwYJj81MJevaL5IHQ==", "engines": { "node": ">=20.0.0" } diff --git a/portal-mcpl/package-lock.json b/portal-mcpl/package-lock.json index 5f43fe3..eee5229 100644 --- a/portal-mcpl/package-lock.json +++ b/portal-mcpl/package-lock.json @@ -1,14 +1,14 @@ { "name": "@animalabs/portal-mcpl", - "version": "0.1.1", + "version": "0.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@animalabs/portal-mcpl", - "version": "0.1.1", + "version": "0.2.0", "dependencies": { - "@animalabs/mcpl-core": "^0.1.0", + "@animalabs/mcpl-core": "^0.2.0", "@animalabs/portal-client": "file:../portal-client", "@animalabs/portal-protocol": "file:../portal-protocol" }, @@ -54,9 +54,9 @@ }, "../portal-client": { "name": "@animalabs/portal-client", - "version": "0.1.0", + "version": "0.3.0", "dependencies": { - "@animalabs/portal-protocol": "^0.1.0", + "@animalabs/portal-protocol": "^0.3.0", "ws": "^8.18.0" }, "devDependencies": { @@ -71,7 +71,7 @@ }, "../portal-protocol": { "name": "@animalabs/portal-protocol", - "version": "0.1.0", + "version": "0.3.0", "devDependencies": { "@types/node": "^20.0.0", "tsx": "^4.7.0", @@ -82,9 +82,9 @@ } }, "node_modules/@animalabs/mcpl-core": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/@animalabs/mcpl-core/-/mcpl-core-0.1.0.tgz", - "integrity": "sha512-V0d9/0K6yMamgThJi6pJtDt1tOGyBlZw88xF/RC7u7Bj42SNGxN7/n3Bx34DoanYcqOz5049iogqGRbTTi5tTg==", + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@animalabs/mcpl-core/-/mcpl-core-0.2.0.tgz", + "integrity": "sha512-n8ENEC6GmoJE1nWc4q7JLUnpB+Z/pz3O36XAMvkhgCAdRUwtlr3mKnPegZ52UZabK8gKDvWWDWYdZhVKPayQaw==", "license": "MIT", "engines": { "node": ">=20.0.0" diff --git a/portal-mcpl/src/agent-state.ts b/portal-mcpl/src/agent-state.ts index 70ed495..f60b8bf 100644 --- a/portal-mcpl/src/agent-state.ts +++ b/portal-mcpl/src/agent-state.ts @@ -32,6 +32,8 @@ interface SerializedState { subscriptions?: string[]; /** Channels opted into live reaction visibility (durable, default off). */ reactionChannels?: string[]; + /** Channels opted into inline audio delivery (durable, default off). */ + audioChannels?: string[]; } const PREVIEW_LEN = 140; @@ -50,6 +52,10 @@ export class AgentState { * Reactions from these channels surface in context but NEVER wake the agent * (tagged `chat:reaction`, which matches no wake policy). Durable. */ private reactionChannels = new Set(); + /** Channels opted into inline audio (per-channel, default off). When ON, + * audio attachments from the channel are fetched and delivered as MCPL + * audio blocks instead of URL notes. Durable. */ + private audioChannels = new Set(); private listeners: Array<() => void> = []; /** Notify on any persistence-relevant change (watermark / ping / subscription). */ @@ -124,6 +130,26 @@ export class AgentState { return [...this.reactionChannels]; } + // ── Inline audio (durable, per-channel opt-in; default off) ── + + /** Opt a channel in/out of inline audio delivery. Returns true if changed. */ + setAudioVisibility(channelId: string, enabled: boolean): boolean { + const changed = enabled ? !this.audioChannels.has(channelId) : this.audioChannels.has(channelId); + if (!changed) return false; + if (enabled) this.audioChannels.add(channelId); + else this.audioChannels.delete(channelId); + this.emitChange(); + return true; + } + + isAudioVisible(channelId: string): boolean { + return this.audioChannels.has(channelId); + } + + audioVisibilityList(): string[] { + return [...this.audioChannels]; + } + /** Advance the watermark for a channel (optionally only up to a message), * clearing unseen + pending pings at/under that point. */ markRead(channelId: string, uptoCreatedAt?: string): void { @@ -195,6 +221,7 @@ export class AgentState { pings: this.pings, subscriptions: [...this.subscriptions], reactionChannels: [...this.reactionChannels], + audioChannels: [...this.audioChannels], }; } @@ -204,6 +231,7 @@ export class AgentState { s.pings = data.pings ?? []; s.subscriptions = new Set(data.subscriptions ?? []); s.reactionChannels = new Set(data.reactionChannels ?? []); + s.audioChannels = new Set(data.audioChannels ?? []); return s; } } diff --git a/portal-mcpl/src/agent.ts b/portal-mcpl/src/agent.ts index 94deb7c..4811088 100644 --- a/portal-mcpl/src/agent.ts +++ b/portal-mcpl/src/agent.ts @@ -186,6 +186,16 @@ export class PortalAgent { ? `Reaction visibility ON for ${channelId}. Reactions there now appear in your context as they happen (they never wake you). Subscribe to the channel if you aren't already, so its reactions reach you.` : `Reaction visibility OFF for ${channelId}.`; } + case 'set_audio_visibility': { + const channelId = str(args.channelId); + const enabled = Boolean(args.enabled); + // Durable per-channel opt-in; persisted via onChange. Content-shaping + // only β€” delivery/wake behavior is unchanged (see server.ts buildContent). + this.state.setAudioVisibility(channelId, enabled); + return enabled + ? `Inline audio ON for ${channelId}. Audio attachments there now arrive as playable audio blocks (oversized clips still degrade to URL notes).` + : `Inline audio OFF for ${channelId}.`; + } case 'list_pins': return this.client.call('list_pins', { channelId: str(args.channelId) }); case 'get_pending_pings': diff --git a/portal-mcpl/src/server.ts b/portal-mcpl/src/server.ts index dbc1eb9..037eaff 100644 --- a/portal-mcpl/src/server.ts +++ b/portal-mcpl/src/server.ts @@ -259,7 +259,21 @@ export class PortalMcplServer { const meta = wakeMetadata(message, addressedToMe, reasons); const tags = deriveTags(message, addressedToMe, reasons); const channelMcplId = portalChannelId(message.channelId); - void buildContent(message).then((content) => { + const inlineAudio = this.agent.state.isAudioVisible(message.channelId); + // A clip too large to inline can't be heard by anyone β€” flag it in Discord + // with a native 🐘 so the sender knows (idempotent: the shared bot reacts + // once even if several opted-in agents notice it). + if (inlineAudio) { + for (const att of message.attachments) { + if (audioMimeFor(att) && att.size > AUDIO_INLINE_CAP) { + void this.client.react(message.id, '🐘', false, true).catch((err: Error) => { + console.error(`[portal-mcpl] oversized-audio flag failed (needs ADD_REACTIONS): ${err.message}`); + }); + break; + } + } + } + void buildContent(message, { inlineAudio }).then((content) => { if (this.openChannels.has(message.channelId)) { conn .sendRequest(method.CHANNELS_INCOMING, { @@ -463,7 +477,7 @@ function deriveTags( for (const a of message.attachments ?? []) { const ct = (a.contentType ?? '').toLowerCase(); if (ct.startsWith('image/')) t.add('chat:has-image'); - else if (ct.startsWith('audio/')) t.add('chat:has-audio'); + else if (audioMimeFor(a)) t.add('chat:has-audio'); else t.add('chat:has-file'); } if (message.threadId) t.add('chat:thread'); @@ -484,30 +498,100 @@ function authorOf(message: PortalMessage): { id: string; name: string } { /** Max image bytes to fetch + inline as a vision block. */ const IMAGE_INLINE_CAP = 5 * 1024 * 1024; +/** Max audio bytes to fetch + inline as an audio block (raw; ~16MB as base64, + * under typical provider inline-media ceilings with headroom for images). */ +const AUDIO_INLINE_CAP = 12 * 1024 * 1024; +/** Max audio blocks inlined per message (each is large in tokens/bytes). */ +const MAX_AUDIO_PER_MESSAGE = 2; + +/** Extension β†’ MIME fallback: Discord's attachment `contentType` is optional. */ +const AUDIO_EXTENSION_MIME: Record = { + mp3: 'audio/mp3', + wav: 'audio/wav', + ogg: 'audio/ogg', + oga: 'audio/ogg', + opus: 'audio/opus', + flac: 'audio/flac', + aac: 'audio/aac', + m4a: 'audio/mp4', + aiff: 'audio/aiff', + aif: 'audio/aiff', +}; + +/** Collapse MP3 MIME aliases (Discord reports `audio/mpeg`; legacy uploaders + * use `audio/mpeg3`/`audio/x-mpeg-3`) to the widely-expected `audio/mp3`. */ +export function normalizeAudioMime(mime: string): string { + const bare = mime.split(';')[0].trim().toLowerCase(); + return ['audio/mpeg', 'audio/mpg', 'audio/mpeg3', 'audio/x-mpeg-3'].includes(bare) + ? 'audio/mp3' + : bare; +} + +/** Resolve an attachment's audio MIME: content-type first, else extension. + * Returns undefined when the attachment isn't audio (or isn't recognizable). */ +export function audioMimeFor(att: { name: string; contentType: string | null }): string | undefined { + const ct = (att.contentType ?? '').toLowerCase(); + if (ct.startsWith('audio/')) return normalizeAudioMime(ct); + const ext = att.name.split('.').pop()?.toLowerCase() ?? ''; + return AUDIO_EXTENSION_MIME[ext]; +} + +/** Fetch an attachment body with a bounded timeout. */ +async function fetchAttachment(url: string, timeoutMs = 15000): Promise { + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), timeoutMs); + const res = await fetch(url, { signal: ctrl.signal }).finally(() => clearTimeout(timer)); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + return Buffer.from(await res.arrayBuffer()); +} + +function attachmentNote(att: PortalMessage['attachments'][number]): string { + const dur = att.duration !== undefined ? `, ${Math.round(att.duration)}s` : ''; + return `[attachment "${att.name}" (${att.contentType ?? 'unknown'}, ${att.size}B${dur}) β€” ${att.url}]`; +} /** Build MCPL content blocks for a message: the text line, plus inlined image - * attachments (so the agent can actually see them) and notes for the rest. + * attachments (so the agent can actually see them), inlined audio when the + * channel is opted in (so it can hear them), and notes for the rest. * Best-effort β€” a failed fetch degrades to a text note, never drops the msg. */ -async function buildContent(m: PortalMessage): Promise { - const blocks: ContentBlock[] = [textContent(render(m))]; - for (const att of m.attachments) { +async function buildContent( + m: PortalMessage, + opts: { inlineAudio?: boolean } = {}, +): Promise { + // Decide each attachment's treatment synchronously (audio slots are ordered), + // then fetch the inlined ones concurrently β€” outcomes are independent. + let audioSlots = MAX_AUDIO_PER_MESSAGE; + const jobs: Promise[] = m.attachments.map((att) => { const ct = (att.contentType ?? '').toLowerCase(); + const audioMime = opts.inlineAudio ? audioMimeFor(att) : undefined; if (ct.startsWith('image/') && att.size > 0 && att.size <= IMAGE_INLINE_CAP) { - try { - const ctrl = new AbortController(); - const timer = setTimeout(() => ctrl.abort(), 15000); - const res = await fetch(att.url, { signal: ctrl.signal }).finally(() => clearTimeout(timer)); - if (!res.ok) throw new Error(`HTTP ${res.status}`); - const data = Buffer.from(await res.arrayBuffer()).toString('base64'); - blocks.push({ type: 'image', data, mimeType: ct }); - } catch (err) { - blocks.push(textContent(`[image "${att.name}" unavailable: ${(err as Error).message} β€” ${att.url}]`)); - } - } else if (m.attachments.length) { - blocks.push(textContent(`[attachment "${att.name}" (${att.contentType ?? 'unknown'}, ${att.size}B) β€” ${att.url}]`)); + return fetchAttachment(att.url).then( + (buf): ContentBlock => ({ type: 'image', data: buf.toString('base64'), mimeType: ct }), + (err: Error) => textContent(`[image "${att.name}" unavailable: ${err.message} β€” ${att.url}]`), + ); } - } - return blocks; + if (audioMime && att.size > 0 && att.size <= AUDIO_INLINE_CAP && audioSlots > 0) { + audioSlots--; + return fetchAttachment(att.url).then( + (buf): ContentBlock => ({ type: 'audio', data: buf.toString('base64'), mimeType: audioMime }), + (err: Error) => textContent(`[audio "${att.name}" unavailable: ${err.message} β€” ${att.url}]`), + ); + } + // Audio that WOULD inline but can't β€” say why, so the agent can tell it + // apart from a generic file. + if (audioMime && att.size > AUDIO_INLINE_CAP) { + return Promise.resolve( + textContent(`[audio "${att.name}" too large to inline (${att.size}B > ${AUDIO_INLINE_CAP}B) β€” ${att.url}]`), + ); + } + if (audioMime && audioSlots <= 0) { + return Promise.resolve( + textContent(`[audio "${att.name}" not inlined (max ${MAX_AUDIO_PER_MESSAGE} per message) β€” ${att.url}]`), + ); + } + return Promise.resolve(textContent(attachmentNote(att))); + }); + return [textContent(render(m)), ...(await Promise.all(jobs))]; } function render(m: PortalMessage): string { diff --git a/portal-mcpl/src/tools.ts b/portal-mcpl/src/tools.ts index ab96950..09c43f6 100644 --- a/portal-mcpl/src/tools.ts +++ b/portal-mcpl/src/tools.ts @@ -159,6 +159,23 @@ export const toolDefinitions: ToolDefinition[] = [ required: ['channelId', 'visible'], }, }, + { + name: 'set_audio_visibility', + description: + 'Opt a channel in or out of inline audio. When ON, audio attachments ' + + '(voice messages, audio files) from that channel are fetched and delivered ' + + 'to you as playable audio blocks instead of URL notes β€” use it only if your ' + + 'model can hear audio. Oversized clips (>12MB) still arrive as URL notes. ' + + 'Default OFF, persisted across restarts.', + inputSchema: { + type: 'object', + properties: { + channelId: { type: 'string', description: PORTAL_CHANNEL_ID_DESC }, + enabled: { type: 'boolean', description: 'true = deliver audio from this channel inline; false = stop.' }, + }, + required: ['channelId', 'enabled'], + }, + }, { name: 'fetch_history', description: diff --git a/portal-mcpl/test/audio.test.ts b/portal-mcpl/test/audio.test.ts new file mode 100644 index 0000000..1092999 --- /dev/null +++ b/portal-mcpl/test/audio.test.ts @@ -0,0 +1,46 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { AgentState } from '../src/agent-state.js'; +import { audioMimeFor, normalizeAudioMime } from '../src/server.js'; + +test('audio visibility: per-channel toggle, default off', () => { + const s = new AgentState(); + assert.equal(s.isAudioVisible('c1'), false); + assert.equal(s.setAudioVisibility('c1', true), true); + assert.equal(s.setAudioVisibility('c1', true), false); // no-op + assert.equal(s.isAudioVisible('c1'), true); + assert.equal(s.isAudioVisible('c2'), false); + assert.equal(s.setAudioVisibility('c1', false), true); + assert.equal(s.isAudioVisible('c1'), false); +}); + +test('audio visibility survives serialization', () => { + const s = new AgentState(); + s.setAudioVisibility('c1', true); + const restored = AgentState.fromJSON(JSON.parse(JSON.stringify(s.toJSON()))); + assert.equal(restored.isAudioVisible('c1'), true); + assert.deepEqual(restored.audioVisibilityList(), ['c1']); +}); + +test('normalizeAudioMime collapses MP3 aliases and strips parameters', () => { + assert.equal(normalizeAudioMime('audio/mpeg'), 'audio/mp3'); + assert.equal(normalizeAudioMime('audio/mpg'), 'audio/mp3'); + assert.equal(normalizeAudioMime('audio/mpeg3'), 'audio/mp3'); + assert.equal(normalizeAudioMime('audio/x-mpeg-3'), 'audio/mp3'); + assert.equal(normalizeAudioMime('audio/mpeg; rate=16000'), 'audio/mp3'); + assert.equal(normalizeAudioMime('Audio/OGG'), 'audio/ogg'); + assert.equal(normalizeAudioMime('audio/wav'), 'audio/wav'); +}); + +test('audioMimeFor: content-type first, extension fallback, undefined otherwise', () => { + assert.equal(audioMimeFor({ name: 'clip.bin', contentType: 'audio/mpeg' }), 'audio/mp3'); + // Discord voice messages: audio/ogg with codec parameter + assert.equal(audioMimeFor({ name: 'voice-message.ogg', contentType: 'audio/ogg; codecs=opus' }), 'audio/ogg'); + // contentType missing β†’ extension fallback + assert.equal(audioMimeFor({ name: 'song.mp3', contentType: null }), 'audio/mp3'); + assert.equal(audioMimeFor({ name: 'take.M4A', contentType: null }), 'audio/mp4'); + assert.equal(audioMimeFor({ name: 'sound.flac', contentType: 'application/octet-stream' }), 'audio/flac'); + // not audio + assert.equal(audioMimeFor({ name: 'doc.pdf', contentType: 'application/pdf' }), undefined); + assert.equal(audioMimeFor({ name: 'noext', contentType: null }), undefined); +}); diff --git a/portal-protocol/package-lock.json b/portal-protocol/package-lock.json index 14b902b..976600b 100644 --- a/portal-protocol/package-lock.json +++ b/portal-protocol/package-lock.json @@ -1,12 +1,12 @@ { "name": "@animalabs/portal-protocol", - "version": "0.2.0", + "version": "0.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@animalabs/portal-protocol", - "version": "0.2.0", + "version": "0.3.0", "devDependencies": { "@types/node": "^20.0.0", "tsx": "^4.7.0", diff --git a/portal-protocol/src/message.ts b/portal-protocol/src/message.ts index 54580ac..6709af2 100644 --- a/portal-protocol/src/message.ts +++ b/portal-protocol/src/message.ts @@ -15,6 +15,10 @@ export interface PortalAttachment { url: string; contentType: string | null; size: number; + /** Voice-message clip length in seconds (Discord voice messages only). */ + duration?: number; + /** Base64 amplitude preview (Discord voice messages only). */ + waveform?: string; } /** @@ -82,4 +86,8 @@ export interface PortalMessage { /** ISO-8601 strings β€” the wire carries no Date objects. */ createdAt: string; editedAt?: string; + /** Set on the 2nd..Nth Discord messages of a relay-split over-long send: + * the relay id of the FIRST part. Lets clients group the parts back into + * one logical message. */ + partOf?: RelayMessageId; } diff --git a/portal-protocol/src/rpc.ts b/portal-protocol/src/rpc.ts index 299e113..6d57a68 100644 --- a/portal-protocol/src/rpc.ts +++ b/portal-protocol/src/rpc.ts @@ -44,7 +44,12 @@ export interface SendMessageParams { mentionPersonaIds?: PersonaId[]; } export interface SendMessageResult { + /** The sent message β€” the FIRST part when the relay split an over-long send. */ messageId: RelayMessageId; + /** All parts, in channel order, when `content` exceeded Discord's 2000-char + * limit and the relay split it (markdown-preserving). Absent for a single + * message. Editing/deleting `messageId` operates on the whole set. */ + messageIds?: RelayMessageId[]; } export interface EditMessageParams { diff --git a/portal-relay/package-lock.json b/portal-relay/package-lock.json index 2e49a88..e4ef3bc 100644 --- a/portal-relay/package-lock.json +++ b/portal-relay/package-lock.json @@ -1,12 +1,12 @@ { "name": "@animalabs/portal-relay", - "version": "0.1.1", + "version": "0.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@animalabs/portal-relay", - "version": "0.1.1", + "version": "0.3.0", "dependencies": { "@animalabs/portal-protocol": "file:../portal-protocol", "discord.js": "^14.16.0", @@ -27,7 +27,7 @@ }, "../portal-protocol": { "name": "@animalabs/portal-protocol", - "version": "0.1.0", + "version": "0.3.0", "devDependencies": { "@types/node": "^20.0.0", "tsx": "^4.7.0", diff --git a/portal-relay/src/discord-bot.ts b/portal-relay/src/discord-bot.ts index 96a19a6..996b944 100644 --- a/portal-relay/src/discord-bot.ts +++ b/portal-relay/src/discord-bot.ts @@ -48,6 +48,10 @@ export interface IncomingAttachment { url: string; contentType: string | null; size: number; + /** Voice-message clip length in seconds (Discord voice messages only). */ + duration?: number; + /** Base64 amplitude preview (Discord voice messages only). */ + waveform?: string; } /** A reaction summary as carried on a fetched/converted message (counts only; @@ -815,6 +819,8 @@ export class DiscordBot implements WebhookOps, RoleOps { url: a.url, contentType: a.contentType ?? null, size: a.size ?? 0, + duration: a.duration ?? undefined, + waveform: a.waveform ?? undefined, })) ?? [], reactions: [...(msg.reactions?.cache?.values() ?? [])].map((r) => ({ emoji: r.emoji.id ? `${r.emoji.name}:${r.emoji.id}` : (r.emoji.name ?? '?'), diff --git a/portal-relay/src/discord-markdown.ts b/portal-relay/src/discord-markdown.ts new file mode 100644 index 0000000..8f51c40 --- /dev/null +++ b/portal-relay/src/discord-markdown.ts @@ -0,0 +1,769 @@ +/** + * Discord-markdown fence/emphasis balancer. + * + * Discord parses each message independently, so a markdown construct that + * straddles a message boundary breaks (a half-open code fence corrupts + * everything after it). This splits text so any construct open at a chunk + * boundary is closed at the chunk end and reopened at the next, and reports the + * open-construct stack so continuity can be threaded across separate sends. + * + * Scope: code fences (``` / ~~~), inline code, emphasis (*, **, ***, _, __, ~~, + * ||). Pure and dependency-free. + * Ported from chapterx (fix/markdown-across-message-splits) β€” keep in sync upstream. + */ + +export interface OpenMark { + kind: + 'fence' | 'inlineCode' | 'bold' | 'italic' | 'boldItalic' | 'underline' | 'strike' | 'spoiler'; + /** Verbatim opener, e.g. "```bash", "~~~", "``", "**". */ + opener: string; + /** Closer to emit. Fence: marker run only (no info string), on its own line. */ + closer: string; +} + +/** + * The open-construct stack at a boundary. Index 0 is the OUTERMOST construct. + * A `fence` or `inlineCode` is exclusive β€” when one is open, no inline parsing + * happens inside it, so it is always the sole entry on the stack. + */ +export type MarkdownCarry = OpenMark[]; + +/** A ready-to-send chunk plus the exact synthetic strings injected into it. */ +export interface ChunkPiece { + /** Text to send (inherited reopener prepended, synthetic closer appended). */ + text: string; + /** Synthetic reopener prepended to `text` (for later precise stripping). */ + bridgeOpen?: string; + /** Synthetic closer appended to `text` (for later precise stripping). */ + bridgeClose?: string; +} + +const DELIM_CHARS = new Set(['`', '*', '_', '~', '|', '\\']); + +function isWordChar(ch: string | undefined): boolean { + return ch !== undefined && /[A-Za-z0-9_]/.test(ch); +} + +function isWhitespace(ch: string | undefined): boolean { + return ch === undefined || /\s/.test(ch); +} + +/** True if the text contains no markdown delimiter characters at all. */ +function hasNoDelimiters(text: string): boolean { + for (const ch of text) { + if (DELIM_CHARS.has(ch)) return false; + } + return true; +} + +// --------------------------------------------------------------------------- +// Parsing +// --------------------------------------------------------------------------- + +interface Construct { + kind: OpenMark['kind']; + opener: string; + closer: string; + /** Index of the first opener char (βˆ’1 for an inherited startCarry construct). */ + openStart: number; + /** Index just past the opener (content start). 0 for inherited carry. */ + contentStart: number; + /** Index of the first closer char (= content end). text.length if unclosed. */ + contentEnd: number; + /** Index just past the closer. text.length if unclosed. */ + closeEnd: number; +} + +interface ParseResult { + /** Matched (closed) constructs, plus any inherited carry construct. */ + constructs: Construct[]; + /** Constructs still open at the very end of the text (outermost first). */ + endStack: OpenMark[]; +} + +const EMPHASIS_BY_CHAR: Record> = { + '*': { 1: 'italic', 2: 'bold', 3: 'boldItalic' }, + _: { 1: 'italic', 2: 'underline' }, + '~': { 2: 'strike' }, + '|': { 2: 'spoiler' }, +}; + +/** Pending open emphasis delimiter awaiting a match. */ +interface PendingEmphasis { + char: string; + len: number; + start: number; // index of first delimiter char + end: number; // index just past the run + canOpen: boolean; + canClose: boolean; + kind: OpenMark['kind']; +} + +/** + * Parse `text` into matched constructs and the open stack at the end. + * + * `startCarry` represents constructs already open before `text` begins (e.g. + * inherited from a previous message). The innermost inherited construct, if it + * is exclusive (fence/inlineCode), means the text begins inside that construct. + */ +function parse(text: string, startCarry: MarkdownCarry = []): ParseResult { + const constructs: Construct[] = []; + + // Model inherited carry: if the innermost is exclusive, we begin inside it + // and must first find its closer before any other parsing resumes. + const inherited = startCarry.length > 0 ? startCarry[startCarry.length - 1]! : undefined; + const inheritedIsExclusive = + inherited !== undefined && (inherited.kind === 'fence' || inherited.kind === 'inlineCode'); + + // Track inherited constructs as open spans starting "before" the text. + // Outer carry entries (non-exclusive) wrap the whole text; the exclusive + // innermost one closes when its closer appears. + const inheritedConstructs: Construct[] = startCarry.map((m) => ({ + kind: m.kind, + opener: m.opener, + closer: m.closer, + openStart: -1, + contentStart: 0, + contentEnd: text.length, + closeEnd: text.length, + })); + + const lines = splitLinesWithIndex(text); + + // Emphasis delimiter stack (only used outside exclusive constructs). + const emphasisStack: PendingEmphasis[] = []; + + // Exclusive state: a currently-open fence or inline-code span. + let exclusive: + | { + kind: 'fence'; + char: string; + runLen: number; + opener: string; + openStart: number; + contentStart: number; + } + | { + kind: 'inlineCode'; + runLen: number; + opener: string; + openStart: number; + contentStart: number; + } + | undefined; + + // Seed exclusive state from inherited carry so the text starts inside it. + if (inheritedIsExclusive && inherited) { + if (inherited.kind === 'fence') { + const { char, runLen } = parseFenceMarker(inherited.opener); + exclusive = { + kind: 'fence', + char, + runLen, + opener: inherited.opener, + openStart: -1, + contentStart: 0, + }; + } else { + const runLen = inherited.opener.length; + exclusive = { + kind: 'inlineCode', + runLen, + opener: inherited.opener, + openStart: -1, + contentStart: 0, + }; + } + } + + for (const line of lines) { + const lineText = line.text; + const lineStart = line.start; + + // ----- Inside an open fence: only a matching closer line ends it. ----- + if (exclusive && exclusive.kind === 'fence') { + const fenceClose = matchFenceClose(lineText, exclusive.char, exclusive.runLen); + if (fenceClose) { + const contentEnd = lineStart + fenceClose.markerOffset; + const closeEnd = lineStart + lineText.length; + recordExclusive(exclusive, contentEnd, closeEnd); + exclusive = undefined; + } + continue; + } + + // ----- Not inside a fence: scan the line char by char. ----- + let i = 0; + while (i < lineText.length) { + const absIdx = lineStart + i; + const ch = lineText[i]!; + + // Backslash escape: the next char is literal. + if (ch === '\\' && !exclusive) { + i += 2; + continue; + } + + // Inside inline code: only a matching backtick run closes it. + if (exclusive && exclusive.kind === 'inlineCode') { + if (ch === '`') { + const run = backtickRunLength(lineText, i); + if (run === exclusive.runLen) { + const contentEnd = absIdx; + const closeEnd = absIdx + run; + recordExclusive(exclusive, contentEnd, closeEnd); + exclusive = undefined; + i += run; + continue; + } + i += run; + continue; + } + i += 1; + continue; + } + + // Fence opener? Only valid at line start (after up to 3 spaces). + if (ch === '`' || ch === '~') { + if (i === 0) { + const fence = tryFenceOpen(lineText); + if (fence) { + exclusive = { + kind: 'fence', + char: fence.char, + runLen: fence.runLen, + opener: fence.opener, + openStart: absIdx, + contentStart: lineStart + lineText.length, // content begins next line + }; + i = lineText.length; // rest of opener line is the info string + continue; + } + } + // Leading-whitespace fence (1–3 spaces of indent). + if (i <= 3 && lineText.slice(0, i).trim() === '') { + const fence = tryFenceOpen(lineText.slice(i)); + if (fence) { + exclusive = { + kind: 'fence', + char: fence.char, + runLen: fence.runLen, + opener: fence.opener, + openStart: absIdx, + contentStart: lineStart + lineText.length, + }; + i = lineText.length; + continue; + } + } + } + + // Inline code opener (backtick run, mid-line or non-fence). + if (ch === '`') { + const run = backtickRunLength(lineText, i); + exclusive = { + kind: 'inlineCode', + runLen: run, + opener: '`'.repeat(run), + openStart: absIdx, + contentStart: absIdx + run, + }; + i += run; + continue; + } + + // Emphasis delimiters. + if (ch === '*' || ch === '_' || ch === '~' || ch === '|') { + const run = runLength(lineText, i, ch); + const handled = handleEmphasisRun(text, absIdx, run, ch, emphasisStack, constructs); + i += handled; + continue; + } + + i += 1; + } + } + + // Anything still exclusive at end is an unclosed fence/inline β†’ open at end. + const endStack: OpenMark[] = []; + + // Outer inherited (non-exclusive) carry entries remain open if never closed. + // For simplicity inherited non-exclusive carry is treated as wrapping the + // whole text and remaining open (it is dropped by exclusiveOnly at the cross + // call boundary anyway). They are added to constructs so split points inside + // see them as straddling. + for (const ic of inheritedConstructs) { + if (ic.kind !== 'fence' && ic.kind !== 'inlineCode') { + constructs.push(ic); + endStack.push({ kind: ic.kind, opener: ic.opener, closer: ic.closer }); + } + } + + if (exclusive) { + // Unclosed fence/inline: record as a construct open through end-of-text. + constructs.push({ + kind: exclusive.kind, + opener: exclusive.opener, + closer: exclusive.kind === 'fence' ? fenceCloserFor(exclusive.opener) : exclusive.opener, + openStart: exclusive.openStart, + contentStart: exclusive.contentStart, + contentEnd: text.length, + closeEnd: text.length, + }); + endStack.push({ + kind: exclusive.kind, + opener: exclusive.opener, + closer: exclusive.kind === 'fence' ? fenceCloserFor(exclusive.opener) : exclusive.opener, + }); + } + + // Unmatched emphasis delimiters left on the stack are open at end ONLY if + // exclusive isn't set (already handled). They render literally if never + // closed, so they are open-at-end but should not bridge internal splits + // (they are not in `constructs`). We still report them in endStack so the + // caller can decide (the loop filters to exclusiveOnly for cross-call carry). + for (const pend of emphasisStack) { + endStack.push({ + kind: pend.kind, + opener: pend.char.repeat(pend.len), + closer: pend.char.repeat(pend.len), + }); + } + + return { constructs, endStack }; + + // --- inner helpers that close over `constructs` --- + function recordExclusive( + ex: NonNullable, + contentEnd: number, + closeEnd: number, + ): void { + constructs.push({ + kind: ex.kind, + opener: ex.opener, + closer: ex.kind === 'fence' ? fenceCloserFor(ex.opener) : ex.opener, + openStart: ex.openStart, + contentStart: ex.contentStart, + contentEnd, + closeEnd, + }); + } +} + +/** + * Process an emphasis delimiter run; returns how many characters to advance. + * Pushes openers and records matched constructs on close. + */ +function handleEmphasisRun( + text: string, + start: number, + len: number, + char: string, + stack: PendingEmphasis[], + constructs: Construct[], +): number { + const before = start > 0 ? text[start - 1] : undefined; + const after = start + len < text.length ? text[start + len] : undefined; + + // Flanking rules (simplified CommonMark / Discord). + let canOpen = !isWhitespace(after); + let canClose = !isWhitespace(before); + if (char === '_') { + // Underscore emphasis does not work intra-word. + canOpen = canOpen && !isWordChar(before); + canClose = canClose && !isWordChar(after); + } + + const kind = EMPHASIS_BY_CHAR[char]?.[len]; + // ~ and | only form constructs at length 2; a length-1 ~ or | is literal. + if (!kind) { + return len; + } + + // Try to close a matching open delimiter of the same char and length. + if (canClose) { + for (let s = stack.length - 1; s >= 0; s--) { + const open = stack[s]!; + if (open.char === char && open.len === len && open.canOpen) { + // Record the matched construct. + constructs.push({ + kind, + opener: char.repeat(len), + closer: char.repeat(len), + openStart: open.start, + contentStart: open.end, + contentEnd: start, + closeEnd: start + len, + }); + // Discard delimiters opened between the match (unmatched/literal). + stack.length = s; + return len; + } + } + } + + if (canOpen) { + stack.push({ char, len, start, end: start + len, canOpen, canClose, kind }); + } + return len; +} + +// --------------------------------------------------------------------------- +// Fence helpers +// --------------------------------------------------------------------------- + +interface LineWithIndex { + text: string; + start: number; +} + +function splitLinesWithIndex(text: string): LineWithIndex[] { + const lines: LineWithIndex[] = []; + let start = 0; + for (let i = 0; i <= text.length; i++) { + if (i === text.length || text[i] === '\n') { + lines.push({ text: text.slice(start, i), start }); + start = i + 1; + } + } + return lines; +} + +function runLength(line: string, i: number, ch: string): number { + let n = 0; + while (i + n < line.length && line[i + n] === ch) n++; + return n; +} + +function backtickRunLength(line: string, i: number): number { + return runLength(line, i, '`'); +} + +/** Try to parse a fence opener from the start of a (de-indented) line. */ +function tryFenceOpen(line: string): { char: string; runLen: number; opener: string } | undefined { + const m = /^(`{3,}|~{3,})(.*)$/.exec(line); + if (!m) return undefined; + const marker = m[1]!; + const info = m[2]!; + const char = marker[0]!; + // Backtick fences may not have a backtick in the info string. + if (char === '`' && info.includes('`')) return undefined; + return { char, runLen: marker.length, opener: marker + info.trimEnd() }; +} + +/** Does this line close an open fence with the given marker char/run? */ +function matchFenceClose( + line: string, + char: string, + openRunLen: number, +): { markerOffset: number } | undefined { + const m = /^( {0,3})(`{3,}|~{3,})\s*$/.exec(line); + if (!m) return undefined; + const indent = m[1]!; + const marker = m[2]!; + if (marker[0] !== char) return undefined; + if (marker.length < openRunLen) return undefined; + return { markerOffset: indent.length }; +} + +function parseFenceMarker(opener: string): { char: string; runLen: number } { + const m = /^(`{3,}|~{3,})/.exec(opener); + if (!m) return { char: '`', runLen: 3 }; + return { char: m[1]![0]!, runLen: m[1]!.length }; +} + +/** The closer string for a fence opener: just the marker run, no info. */ +function fenceCloserFor(opener: string): string { + const { char, runLen } = parseFenceMarker(opener); + return char.repeat(runLen); +} + +// --------------------------------------------------------------------------- +// Public stack helpers +// --------------------------------------------------------------------------- + +/** Closing delimiters for an open stack, innermost first (reverse order). */ +export function closeMarks(stack: MarkdownCarry): string { + let out = ''; + for (let i = stack.length - 1; i >= 0; i--) { + const mark = stack[i]!; + if (mark.kind === 'fence') { + out += (out.endsWith('\n') ? '' : '\n') + mark.closer; + } else { + out += mark.closer; + } + } + return out; +} + +/** + * A fence's reopener body: the marker run plus at most the first + * whitespace-delimited word of the info string (the language tag), optionally + * truncated to `infoBudget` chars. Replaying the entire verbatim opening line + * is unnecessary β€” Discord only consumes the first token for syntax + * highlighting β€” and a long info string would otherwise blow past maxLength + * when the fence straddles a cut. Truncating the tag only affects the synthetic + * reopener (a bridge that is stripped on reconstruction), never the body, so + * losslessness is preserved. + */ +function fenceReopener(opener: string, infoBudget = Infinity): string { + const { char, runLen } = parseFenceMarker(opener); + const marker = char.repeat(runLen); + const info = opener.slice(marker.length); + const firstWord = info.trimStart().split(/\s/)[0] ?? ''; + const budget = Math.max(0, infoBudget); + return marker + (firstWord.length > budget ? firstWord.slice(0, budget) : firstWord); +} + +/** Reopening delimiters for an open stack, outermost first. */ +export function reopenMarks(stack: MarkdownCarry): string { + let out = ''; + for (const mark of stack) { + if (mark.kind === 'fence') { + out += fenceReopener(mark.opener) + '\n'; + } else { + out += mark.opener; + } + } + return out; +} + +/** Keep only exclusive constructs (fence / inlineCode) for cross-call carry. */ +export function exclusiveOnly(stack: MarkdownCarry): MarkdownCarry { + return stack.filter((m) => m.kind === 'fence' || m.kind === 'inlineCode'); +} + +/** Open-construct stack at the end of `text`, given inherited `start` carry. */ +export function scanMarkdown(text: string, start: MarkdownCarry = []): MarkdownCarry { + return parse(text, start).endStack; +} + +// --------------------------------------------------------------------------- +// Splitting +// --------------------------------------------------------------------------- + +/** + * Split `text` into chunks each <= maxLength, closing any open construct at a + * chunk boundary and reopening it at the next. `startCarry` continues a + * construct inherited from a previous message; `endCarry` reports the open + * stack after the last chunk so callers can thread continuity across calls. + */ +export function splitPreservingMarkdown( + text: string, + maxLength: number, + startCarry: MarkdownCarry = [], +): { chunks: ChunkPiece[]; endCarry: MarkdownCarry } { + // Fast path: short, no inherited carry, no markdown. + if (startCarry.length === 0 && text.length <= maxLength && hasNoDelimiters(text)) { + return { chunks: [{ text }], endCarry: [] }; + } + + const { constructs, endStack } = parse(text, startCarry); + + // Every recorded construct is bridgeable: matched emphasis (closed), every + // fence/inline-code span (open or closed), and inherited-carry constructs. + // Unmatched emphasis is never recorded in `constructs`, so it is left literal. + const bridgeable = constructs; + + // stackAt(pos): constructs straddling pos (opener before, closer after), + // ordered outermost-first. + const stackAt = (pos: number): OpenMark[] => { + return bridgeable + .filter((c) => c.contentStart <= pos && pos <= c.contentEnd) + .sort((a, b) => a.openStart - b.openStart) + .map((c) => ({ kind: c.kind, opener: c.opener, closer: c.closer })); + }; + + const chunks: ChunkPiece[] = []; + let chunkStart = 0; + let inherited: OpenMark[] = startCarry.slice(); + + // Reopener for an inherited stack, with a fence's info-string tail capped so + // the reopener can never, on its own, crowd out maxLength. A fence is + // exclusive (the sole stack entry when active); the room we must leave is its + // own closer (marker + newline) plus at least one body char. This is what + // makes invariant A (chunk.text.length <= maxLength) a hard guarantee even + // when the original opening line is pathologically long. Emphasis openers are + // tiny and never capped. + const reopenerFor = (stack: OpenMark[]): string => { + let out = ''; + for (const mark of stack) { + if (mark.kind === 'fence') { + const { runLen } = parseFenceMarker(mark.opener); + // marker + info + '\n' (reopener) and marker + '\n' (closer) and 1 body. + const infoBudget = maxLength - runLen - 1 - (runLen + 1) - 1; + out += fenceReopener(mark.opener, infoBudget) + '\n'; + } else { + out += mark.opener; + } + } + return out; + }; + + const pushChunk = (endPos: number, closeStack: OpenMark[], bridgeOpen: string): void => { + const body = text.slice(chunkStart, endPos); + const bridgeClose = closeStack.length ? closeMarksForBody(body, closeStack) : ''; + chunks.push({ + text: bridgeOpen + body + bridgeClose, + ...(bridgeOpen ? { bridgeOpen } : {}), + ...(bridgeClose ? { bridgeClose } : {}), + }); + chunkStart = endPos; + inherited = closeStack; + }; + + // Delimiter chars whose runs merge when concatenated (backslash excluded β€” it + // escapes rather than forming a run). + const isRunChar = (ch: string | undefined): boolean => + ch === '`' || ch === '~' || ch === '*' || ch === '_' || ch === '|'; + + // Does a NON-fence bridgeable construct's content span cross `pos`? Bridging + // an inline construct (inline code / emphasis) across a cut abuts its + // delimiter with the body and can fuse delimiter runs at the seam; a newline + // that lands inside such a construct also needlessly fragments a small span. + // Fences are exempt β€” straddling them at a newline is exactly how they bridge, + // and their newline-anchored markers never fuse. + const straddlesNonFence = (pos: number): boolean => + bridgeable.some((c) => c.kind !== 'fence' && c.contentStart <= pos && pos <= c.contentEnd); + + // Would emitting this chunk (body = text[chunkStart, cut], closed by + // `closeStack`) and reopening `closeStack` in the next chunk fuse a synthetic + // bridge string with an identical run char in the adjacent body, changing how + // Discord parses the seam? Fences are newline-delimited so they never fuse. + const seamMerges = (cut: number, closeStack: OpenMark[], isLast: boolean): boolean => { + const body = text.slice(chunkStart, cut); + if (body.length === 0) return false; + const closer = closeStack.length ? closeMarksForBody(body, closeStack) : ''; + // (ii) synthetic closer starts with the run char the body ends with. + if (closer && isRunChar(closer[0]) && closer[0] === body[body.length - 1]) return true; + if (isLast) return false; + // (i) reopener (inherited by the next chunk) ends with the run char the next + // chunk's body starts with. + const reopener = reopenerFor(closeStack); + if ( + reopener && + isRunChar(reopener[reopener.length - 1]) && + reopener[reopener.length - 1] === text[cut] + ) + return true; + return false; + }; + + while (chunkStart < text.length) { + const reopen = inherited.length ? reopenerFor(inherited) : ''; + const reopenLen = reopen.length; + + // Pick a cut whose real size (reopener + body + the closers for constructs + // open AT THE CUT) fits maxLength. Shrink from the largest feasible end, + // re-deriving the cut and its close-stack each pass so closers added by a + // snapped cut are counted. `cut <= chunkStart + 1` escapes the pathological + // case where the closers alone exceed maxLength. + let end = Math.min(text.length, chunkStart + Math.max(1, maxLength - reopenLen)); + let cut = -1; + let closeStack: OpenMark[] = []; + for (let guard = 0; guard < 64; guard++) { + if (end >= text.length) { + cut = text.length; + // Final chunk: close only exclusive constructs (emphasis stays literal). + closeStack = exclusiveOnly(stackAt(text.length)); + } else { + // Prefer the latest newline within (chunkStart, end] that does not land + // inside a non-fence construct (which would fragment it and risk a seam + // merge); keep it at this chunk's end so reconstruction is exact. + // Otherwise hard-split without bisecting a delimiter run. + const body = text.slice(chunkStart, end); + let nlCut = -1; + for ( + let nl = body.lastIndexOf('\n'); + nl >= 0; + nl = nl > 0 ? body.lastIndexOf('\n', nl - 1) : -1 + ) { + const candidate = chunkStart + nl + 1; + if (!straddlesNonFence(candidate)) { + nlCut = candidate; + break; + } + } + if (nlCut >= 0) { + cut = nlCut; + } else { + cut = avoidDelimiterBisect(end, bridgeable, chunkStart); + if (cut <= chunkStart) cut = end; + } + closeStack = stackAt(cut); + } + + const bodyLen = cut - chunkStart; + const closeLen = closeMarksForBody(text.slice(chunkStart, cut), closeStack).length; + if (reopenLen + bodyLen + closeLen <= maxLength || cut <= chunkStart + 1) break; + end = cut - 1; + if (end <= chunkStart) end = chunkStart + 1; + } + + if (cut < 0) cut = chunkStart + 1; + let isLast = cut >= text.length; + + // Final chunk: if closing the dangling exclusive construct would fuse its + // closer with a trailing delimiter run in the body (e.g. inline code whose + // content ends in backticks), leave it open. It is unclosed in the original + // β€” which Discord already renders literally β€” and there is no more body in + // this call, so leaving it open both avoids the seam and matches the + // original rendering. endCarry still reports it for cross-call continuation. + if (isLast && closeStack.length && seamMerges(cut, closeStack, true)) { + closeStack = []; + } + + // Avoid a seam that would fuse a bridge string with a same-char delimiter run + // in the adjacent body (which merges runs and changes parsing). Pull the cut + // left β€” off the offending delimiter and out of any construct it straddled β€” + // until the seam clears. A smaller body only shrinks the fit, so invariant A + // still holds. Monotonic (cut strictly decreases) so it always terminates. + if (!isLast) { + let guard = 0; + while ( + cut > chunkStart + 1 && + seamMerges(cut, closeStack, isLast) && + guard++ <= text.length + ) { + let next = avoidDelimiterBisect(cut - 1, bridgeable, chunkStart); + if (next <= chunkStart) next = cut - 1; + cut = next; + isLast = cut >= text.length; + closeStack = isLast ? exclusiveOnly(stackAt(text.length)) : stackAt(cut); + } + } + + pushChunk(cut, closeStack, reopen); + if (isLast) break; + } + + if (chunks.length === 0) chunks.push({ text }); + + return { chunks, endCarry: endStack }; +} + +/** Closers for `stack`, ensuring a fence closer starts on its own line. */ +function closeMarksForBody(body: string, stack: MarkdownCarry): string { + let out = ''; + for (let i = stack.length - 1; i >= 0; i--) { + const mark = stack[i]!; + if (mark.kind === 'fence') { + const needsNl = !(out.length ? out.endsWith('\n') : body.endsWith('\n')); + out += (needsNl ? '\n' : '') + mark.closer; + } else { + out += mark.closer; + } + } + return out; +} + +/** Pull `pos` back so it never bisects a delimiter run; never below `floor`. */ +function avoidDelimiterBisect(pos: number, constructs: Construct[], floor: number): number { + for (const c of constructs) { + // Inside an opener run? + if (pos > c.openStart && pos < c.contentStart && c.openStart >= floor) return c.openStart; + // Inside a closer run? + if (pos > c.contentEnd && pos < c.closeEnd && c.contentEnd >= floor) return c.contentEnd; + } + return pos; +} diff --git a/portal-relay/src/message-store.ts b/portal-relay/src/message-store.ts index f9d9ea0..ef0a356 100644 --- a/portal-relay/src/message-store.ts +++ b/portal-relay/src/message-store.ts @@ -31,6 +31,16 @@ export interface MessageRef { personaId?: string; /** Which pooled webhook (id) carried it β€” needed to edit/delete it. */ webhookId?: string; + /** Synthetic markdown reopener the relay prepended when this message is a + * continuation part of a split send (stripped when serving agents). */ + bridgeOpen?: string; + /** Synthetic markdown closer the relay appended at a split boundary. */ + bridgeClose?: string; + /** First part only: Discord msg ids of ALL parts of a split send, in order + * (includes this message). Edit/delete fan out across these. */ + parts?: string[]; + /** Continuation parts only: Discord msg id of the FIRST part. */ + partOf?: string; } /** The message's immediate channel (where `messages.fetch(id)` lives). */ @@ -59,6 +69,10 @@ interface AttributionRow { guildId: string | null; personaId?: string; webhookId?: string; + bridgeOpen?: string; + bridgeClose?: string; + parts?: string[]; + partOf?: string; } export class MessageStore { @@ -87,6 +101,10 @@ export class MessageStore { // Upgrade with authoritative fields a prior (e.g. echo-derived) ref lacked. if (ref.personaId && !existing.personaId) existing.personaId = ref.personaId; if (ref.webhookId && !existing.webhookId) existing.webhookId = ref.webhookId; + if (ref.bridgeOpen && !existing.bridgeOpen) existing.bridgeOpen = ref.bridgeOpen; + if (ref.bridgeClose && !existing.bridgeClose) existing.bridgeClose = ref.bridgeClose; + if (ref.parts && !existing.parts) existing.parts = ref.parts; + if (ref.partOf && !existing.partOf) existing.partOf = ref.partOf; if (existing.personaId) this.persist(existing); return existing; } @@ -125,6 +143,22 @@ export class MessageStore { return ref; } + /** Overwrite split-send metadata on a ref (re-splitting an edit changes the + * bridges, and may shrink or dissolve the part set). Unlike `record()` this + * SETS the fields β€” including back to undefined. */ + setSplitMeta( + discordMsgId: string, + meta: { bridgeOpen?: string; bridgeClose?: string; parts?: string[]; partOf?: string }, + ): void { + const ref = this.getByDiscordId(discordMsgId); + if (!ref) return; + ref.bridgeOpen = meta.bridgeOpen; + ref.bridgeClose = meta.bridgeClose; + ref.parts = meta.parts; + ref.partOf = meta.partOf; + if (ref.personaId) this.persist(ref); + } + /** Relay id for a Discord message, minting a bare ref if unseen. */ ensureForDiscord(discordMsgId: string, derive: () => Omit): MessageRef { return this.getByDiscordId(discordMsgId) ?? this.record(derive()); @@ -161,6 +195,10 @@ export class MessageStore { guildId: ref.guildId, personaId: ref.personaId, webhookId: ref.webhookId, + bridgeOpen: ref.bridgeOpen, + bridgeClose: ref.bridgeClose, + parts: ref.parts, + partOf: ref.partOf, }); while (this.persisted.size > this.persistCap) { const oldest = this.persisted.keys().next().value; diff --git a/portal-relay/src/relay.ts b/portal-relay/src/relay.ts index c7b75c0..dca941a 100644 --- a/portal-relay/src/relay.ts +++ b/portal-relay/src/relay.ts @@ -20,6 +20,7 @@ import type { } from '@animalabs/portal-protocol'; import type { InviteTemplate, PersonaIdentity, PersonaPolicy, RelayConfig, Scope } from './config.js'; import { DiscordBot, type ChannelMeta, type IncomingMessage } from './discord-bot.js'; +import { splitPreservingMarkdown, type ChunkPiece } from './discord-markdown.js'; import { Gateway, type GatewayHooks, Session } from './gateway.js'; import { GuildAllowStore, type GuildAllowChange } from './guild-allowlist.js'; import { HistoryCache } from './history-cache.js'; @@ -30,10 +31,13 @@ import { MirrorCache } from './mirror-cache.js'; import { PermissionsStore, type PermissionChange, computeCapabilities } from './permissions.js'; import { ReadStateStore } from './read-state.js'; import { RolePool } from './role-pool.js'; -import { WebhookPool } from './webhook-pool.js'; +import { PartialSendError, WebhookPool, type WebhookSendOpts } from './webhook-pool.js'; import { AdminServer, type AdminDeps } from './admin/server.js'; import { AuditLog } from './admin/audit.js'; +/** Discord's hard per-message content limit. */ +const DISCORD_MSG_LIMIT = 2000; + export class Relay implements GatewayHooks { private bot: DiscordBot; readonly identity: IdentityStore; @@ -497,23 +501,47 @@ export class Relay implements GatewayHooks { if (!ref.webhookId) throw rpcError('NOT_FOUND', 'no webhook recorded for message'); this.requireCap(personaId, ref.channelId, 'EDIT_OWN'); await this.webhooks.ensureLoaded(ref.channelId); // adopt webhooks post-restart - await this.webhooks.edit(ref.webhookId, ref.discordMsgId, p.content, ref.threadId); + const primary = this.primaryOf(ref); + // All parts of a split went out on one webhook; a rehydrated primary + // missing its webhookId can safely borrow the addressed part's. + if (!primary.webhookId) primary.webhookId = ref.webhookId; + await this.editMessage(primary, p.content); return {}; } case 'delete_message': { const p = params as RpcParams<'delete_message'>; const ref = await this.resolveRef(p.messageId); if (!ref) throw rpcError('NOT_FOUND', 'unknown message'); + const primary = this.primaryOf(ref); + const partIds = primary.parts ?? [primary.discordMsgId]; + let del: (id: string) => Promise; if (ref.personaId === personaId && ref.webhookId) { - // Own webhook message β†’ delete via the webhook. + // Own webhook message β†’ delete via the webhook (every part of a split). this.requireCap(personaId, ref.channelId, 'DELETE_OWN'); await this.webhooks.ensureLoaded(ref.channelId); - await this.webhooks.delete(ref.webhookId, ref.discordMsgId, ref.threadId); + const webhookId = ref.webhookId; + del = (id) => this.webhooks.delete(webhookId, id, ref.threadId); } else { // Someone else's message β†’ moderation delete (bot-level), gated by the // MANAGE_MESSAGES capability (and the bot's Discord Manage Messages perm). this.requireCap(personaId, ref.channelId, 'MANAGE_MESSAGES'); - await this.bot.deleteAnyMessage(ref.threadId ?? ref.channelId, ref.discordMsgId); + del = (id) => this.bot.deleteAnyMessage(ref.threadId ?? ref.channelId, id); + } + // Attempt every part before failing so one bad part doesn't strand the + // rest β€” but a failure must still surface, not report success. + const failed: Error[] = []; + for (const id of partIds) { + try { + await del(id); + } catch (err) { + failed.push(err as Error); + } + } + if (failed.length) { + throw rpcError( + 'DISCORD_ERROR', + `failed to delete ${failed.length}/${partIds.length} message part(s): ${failed[0].message}`, + ); } return {}; } @@ -683,7 +711,7 @@ export class Relay implements GatewayHooks { private async sendMessage( personaId: string, p: RpcParams<'send_message'>, - ): Promise<{ messageId: string }> { + ): Promise<{ messageId: string; messageIds?: string[] }> { const cfg = this.identity.get(personaId)!; const target = await this.bot.resolveTarget(p.channelId); if (!target) throw rpcError('NOT_FOUND', 'channel not found'); @@ -719,23 +747,124 @@ export class Relay implements GatewayHooks { } } - const { messageId, webhookId } = await this.webhooks.send(target.parentChannelId, personaId, { + const base = { threadId: target.threadId, username: cfg.displayName, avatarURL: this.identity.avatarUrl(cfg), - content, - files: p.files, - }); + }; - const ref = this.store.record({ - channelId: target.parentChannelId, - threadId: target.threadId, - guildId, - discordMsgId: messageId, - personaId, - webhookId, - }); - return { messageId: ref.relayId }; + if (content.length <= DISCORD_MSG_LIMIT) { + const { messageId, webhookId } = await this.webhooks.send(target.parentChannelId, personaId, { + ...base, + content, + files: p.files, + }); + const ref = this.store.record({ + channelId: target.parentChannelId, + threadId: target.threadId, + guildId, + discordMsgId: messageId, + personaId, + webhookId, + }); + return { messageId: ref.relayId }; + } + + // Over-long send β†’ markdown-preserving split into several Discord messages. + // Files ride on the last part so they land under the complete text. + const { chunks } = splitPreservingMarkdown(content, DISCORD_MSG_LIMIT); + const optsList: WebhookSendOpts[] = chunks.map((c, i) => ({ + ...base, + content: c.text, + files: i === chunks.length - 1 ? p.files : undefined, + })); + let sent: { messageIds: string[]; webhookId: string }; + try { + sent = await this.webhooks.sendMany(target.parentChannelId, personaId, optsList); + } catch (err) { + // Record whatever DID land so the parts stay attributable/editable, and + // name them in the error so the caller can clean up or continue. + if (err instanceof PartialSendError && err.sentIds.length) { + const partial = this.recordParts(target, guildId, personaId, err.webhookId, err.sentIds, chunks); + throw rpcError( + 'DISCORD_ERROR', + `split send failed after ${err.sentIds.length}/${chunks.length} parts ` + + `(${err.reason.message}); posted parts: ${partial.map((r) => r.relayId).join(', ')} ` + + `β€” delete them before retrying, or send the remainder separately`, + ); + } + throw rpcError('DISCORD_ERROR', `split send failed: ${(err as Error).message}`); + } + const refs = this.recordParts(target, guildId, personaId, sent.webhookId, sent.messageIds, chunks); + return { messageId: refs[0].relayId, messageIds: refs.map((r) => r.relayId) }; + } + + /** Record every part of a split send: bridge strings for later stripping, + * `parts` on the first part, `partOf` back-links on continuations. */ + private recordParts( + target: { parentChannelId: string; threadId?: string }, + guildId: string | null, + personaId: string, + webhookId: string, + discordMsgIds: string[], + chunks: ChunkPiece[], + ): MessageRef[] { + return discordMsgIds.map((discordMsgId, i) => + this.store.record({ + channelId: target.parentChannelId, + threadId: target.threadId, + guildId, + discordMsgId, + personaId, + webhookId, + ...splitMetaFor(discordMsgIds, i, chunks), + }), + ); + } + + /** The first part of a split send (edits/deletes operate on the whole set). + * For a standalone message this is the ref itself. */ + private primaryOf(ref: MessageRef): MessageRef { + if (!ref.partOf) return ref; + return this.store.getByDiscordId(ref.partOf) ?? ref; + } + + /** Edit a (possibly split) persona message. The new content is re-split and + * written across the existing Discord messages; surplus parts are deleted. + * Growing beyond the original part count is refused β€” Discord offers no way + * to insert messages in place. */ + private async editMessage(primary: MessageRef, content: string): Promise { + const partIds = primary.parts ?? [primary.discordMsgId]; + const webhookId = primary.webhookId!; + if (content.length <= DISCORD_MSG_LIMIT && partIds.length === 1) { + await this.webhooks.edit(webhookId, primary.discordMsgId, content, primary.threadId); + return; + } + const { chunks } = splitPreservingMarkdown(content, DISCORD_MSG_LIMIT); + if (chunks.length > partIds.length) { + throw rpcError( + 'INVALID_PARAMS', + `edited content needs ${chunks.length} messages but the original send has ` + + `${partIds.length} β€” shorten it, or delete and resend`, + ); + } + const keptIds = partIds.slice(0, chunks.length); + for (let i = 0; i < chunks.length; i++) { + await this.webhooks.edit(webhookId, partIds[i], chunks[i].text, primary.threadId); + // Update this part's bridge metadata immediately β€” a concurrent + // fetch_history between edits must not strip with stale markers. + this.store.setSplitMeta(partIds[i], splitMetaFor(keptIds, i, chunks)); + } + for (const id of partIds.slice(chunks.length)) { + try { + await this.webhooks.delete(webhookId, id, primary.threadId); + this.store.remove(id); + } catch (err) { + // Keep the ref β€” the message is still live in Discord; dropping the + // store row would orphan it beyond any future edit/delete. + console.error('[portal-relay] surplus part delete failed:', (err as Error).message); + } + } } private async react( @@ -1025,11 +1154,13 @@ export class Relay implements GatewayHooks { threadId: inc.threadId, guildId: inc.guildId, author, - content: inc.content, + // Strip the synthetic bridge markers a split send injected, so agents + // (history, live events, pins) see their original unbroken markdown. + content: stripBridges(inc.content, ref), // Render custom-emoji tokens (<:name:id> / ) down to :name: in // the human-readable field so message text reads legibly for the model. // The raw `content` keeps the full tokens for correlation/round-tripping. - cleanContent: renderCustomEmojis(inc.cleanContent), + cleanContent: renderCustomEmojis(stripBridges(inc.cleanContent, ref)), attachments: inc.attachments, mentions: { personas, @@ -1045,6 +1176,7 @@ export class Relay implements GatewayHooks { by: [], })), createdAt: inc.timestamp.toISOString(), + partOf: ref.partOf ? makeRelayId(ref.threadId ?? ref.channelId, ref.partOf) : undefined, }; return { message, authorPersonaId }; } @@ -1110,8 +1242,47 @@ function rpcError(code: string, message: string): Error & { code: string } { return Object.assign(new Error(message), { code }); } +/** Split-send metadata for part `i` of a part-id set: bridge strings for later + * stripping, `parts` on the first part, `partOf` back-links on continuations. + * Shared by initial sends (recordParts) and re-split edits (setSplitMeta). */ +function splitMetaFor( + ids: string[], + i: number, + chunks: ChunkPiece[], +): Pick { + return { + bridgeOpen: chunks[i]?.bridgeOpen, + bridgeClose: chunks[i]?.bridgeClose, + parts: i === 0 && ids.length > 1 ? [...ids] : undefined, + partOf: i > 0 ? ids[0] : undefined, + }; +} + /** Render custom-emoji tokens ('<:name:id>' / '') down to ':name:' * so message text reads legibly for the model. Unicode emoji are untouched. */ function renderCustomEmojis(text: string): string { return text.replace(//g, (_full, name: string) => `:${name}:`); } + +/** Remove the synthetic reopener/closer a split send injected into this part. + * Exact match first; a fence reopener also tolerates a rewritten info string + * (Discord normalizes mentions/emojis in `cleanContent`) by matching on the + * fence marker run alone and dropping that first line. */ +export function stripBridges(text: string, ref: { bridgeOpen?: string; bridgeClose?: string }): string { + let out = text; + if (ref.bridgeOpen) { + if (out.startsWith(ref.bridgeOpen)) { + out = out.slice(ref.bridgeOpen.length); + } else { + const fence = /^(`{3,}|~{3,})/.exec(ref.bridgeOpen)?.[1]; + if (fence && out.startsWith(fence)) { + const nl = out.indexOf('\n'); + out = nl === -1 ? '' : out.slice(nl + 1); + } + } + } + if (ref.bridgeClose && out.endsWith(ref.bridgeClose)) { + out = out.slice(0, out.length - ref.bridgeClose.length); + } + return out; +} diff --git a/portal-relay/src/webhook-pool.ts b/portal-relay/src/webhook-pool.ts index ae957fb..eb2938e 100644 --- a/portal-relay/src/webhook-pool.ts +++ b/portal-relay/src/webhook-pool.ts @@ -40,6 +40,20 @@ export interface WebhookOps { const MARKER = 'portal:relay'; +/** A multi-part send failed after some parts were already posted. */ +export class PartialSendError extends Error { + constructor( + /** Discord message ids of the parts that DID send, in order. */ + public readonly sentIds: string[], + public readonly webhookId: string, + /** The underlying send failure. */ + public readonly reason: Error, + ) { + super(`part ${sentIds.length + 1} failed: ${reason.message}`); + this.name = 'PartialSendError'; + } +} + interface ChannelPool { ids: string[]; /** Per-webhook tail promise; new sends chain onto it for ordering. */ @@ -92,19 +106,44 @@ export class WebhookPool { personaId: string, opts: WebhookSendOpts, ): Promise<{ messageId: string; webhookId: string }> { + try { + const { messageIds, webhookId } = await this.sendMany(parentChannelId, personaId, [opts]); + return { messageId: messageIds[0], webhookId }; + } catch (err) { + // Single send β€” unwrap to the underlying failure callers always saw. + throw err instanceof PartialSendError ? err.reason : err; + } + } + + /** Send several messages as ONE queue item on the persona's webhook, so the + * parts of a split send stay contiguous (no same-webhook send can land + * between them). On a mid-sequence failure, rejects with a `PartialSendError` + * carrying the ids that DID go out so the caller can still record them. */ + async sendMany( + parentChannelId: string, + personaId: string, + optsList: WebhookSendOpts[], + ): Promise<{ messageIds: string[]; webhookId: string }> { const pool = await this.ensurePool(parentChannelId); const webhookId = this.pick(pool, personaId); + const task = async (): Promise => { + const ids: string[] = []; + for (const opts of optsList) { + try { + ids.push((await this.ops.sendWebhook(webhookId, opts)).messageId); + } catch (err) { + throw new PartialSendError(ids, webhookId, err as Error); + } + } + return ids; + }; const prev = pool.tails.get(webhookId) ?? Promise.resolve(); - const next = prev.then( - () => this.ops.sendWebhook(webhookId, opts), - () => this.ops.sendWebhook(webhookId, opts), // prior failure shouldn't block the queue - ); + const next = prev.then(task, task); // prior failure shouldn't block the queue pool.tails.set( webhookId, next.catch(() => undefined), ); - const { messageId } = await next; - return { messageId, webhookId }; + return { messageIds: await next, webhookId }; } /** Adopt a parent channel's webhooks into the cache. Needed before editing or diff --git a/portal-relay/test/discord-markdown.test.ts b/portal-relay/test/discord-markdown.test.ts new file mode 100644 index 0000000..1274c75 --- /dev/null +++ b/portal-relay/test/discord-markdown.test.ts @@ -0,0 +1,1360 @@ +/** + * Exhaustive test suite for src/utils/discord-markdown.ts + * + * Covers: splitPreservingMarkdown, scanMarkdown, closeMarks, reopenMarks, exclusiveOnly. + * + * Organisation: + * 1. Helpers & invariant checkers + * 2. closeMarks / reopenMarks / exclusiveOnly unit tests + * 3. scanMarkdown unit tests + * 4. splitPreservingMarkdown β€” plain-text / fast-path + * 5. splitPreservingMarkdown β€” code fences + * 6. splitPreservingMarkdown β€” inline code + * 7. splitPreservingMarkdown β€” emphasis + * 8. splitPreservingMarkdown β€” nesting & ordering + * 9. splitPreservingMarkdown β€” startCarry continuation + * 10. splitPreservingMarkdown β€” edge cases + */ + +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; + +// Minimal vitest-style expect shim over node:assert (matchers this suite uses). +function deepEq(a: unknown, b: unknown): boolean { + try { + assert.deepStrictEqual(a, b); + return true; + } catch { + return false; + } +} +function makeMatchers(actual: unknown, negate: boolean) { + const check = (cond: boolean, expectation: string) => { + if (negate ? cond : !cond) { + assert.fail( + `expected ${JSON.stringify(actual)}${negate ? ' not' : ''} ${expectation}`, + ); + } + }; + return { + toBe: (exp: unknown) => check(Object.is(actual, exp), `to be ${JSON.stringify(exp)}`), + toEqual: (exp: unknown) => check(deepEq(actual, exp), `to equal ${JSON.stringify(exp)}`), + toContain: (exp: unknown) => + check( + typeof actual === 'string' + ? actual.includes(exp as string) + : Array.isArray(actual) && actual.some((v) => deepEq(v, exp)), + `to contain ${JSON.stringify(exp)}`, + ), + toMatch: (exp: RegExp | string) => + check( + typeof exp === 'string' ? (actual as string).includes(exp) : exp.test(actual as string), + `to match ${exp}`, + ), + toHaveLength: (n: number) => + check((actual as { length: number }).length === n, `to have length ${n}`), + toBeUndefined: () => check(actual === undefined, 'to be undefined'), + toBeGreaterThan: (n: number) => check((actual as number) > n, `to be > ${n}`), + toBeGreaterThanOrEqual: (n: number) => check((actual as number) >= n, `to be >= ${n}`), + toBeLessThan: (n: number) => check((actual as number) < n, `to be < ${n}`), + toBeLessThanOrEqual: (n: number) => check((actual as number) <= n, `to be <= ${n}`), + }; +} +function expect(actual: unknown) { + return { ...makeMatchers(actual, false), not: makeMatchers(actual, true) }; +} +import { + splitPreservingMarkdown, + scanMarkdown, + exclusiveOnly, + closeMarks, + reopenMarks, + type ChunkPiece, + type MarkdownCarry, + type OpenMark, +} from '../src/discord-markdown.js' + +// --------------------------------------------------------------------------- +// 1. Shared helpers +// --------------------------------------------------------------------------- + +/** Strip bridge strings from every chunk and concatenate β€” must equal original. */ +function reconstruct(chunks: ChunkPiece[]): string { + return chunks + .map((c) => { + let t = c.text + if (c.bridgeOpen && t.startsWith(c.bridgeOpen)) t = t.slice(c.bridgeOpen.length) + if (c.bridgeClose && t.endsWith(c.bridgeClose)) t = t.slice(0, t.length - c.bridgeClose.length) + return t + }) + .join('') +} + +/** Every chunk's text must be <= maxLength. */ +function assertAllWithinLimit(chunks: ChunkPiece[], maxLength: number): void { + for (const c of chunks) { + expect(c.text.length).toBeLessThanOrEqual(maxLength) + } +} + +/** Count (non-overlapping) occurrences of needle in haystack. */ +function countOccurrences(haystack: string, needle: string): number { + let n = 0 + let pos = 0 + while ((pos = haystack.indexOf(needle, pos)) !== -1) { + n++ + pos += needle.length + } + return n +} + +// --------------------------------------------------------------------------- +// 2. closeMarks / reopenMarks / exclusiveOnly +// --------------------------------------------------------------------------- + +describe('closeMarks', () => { + it('returns empty string for empty stack', () => { + expect(closeMarks([])).toBe('') + }) + + it('closes a simple fence (adds leading newline)', () => { + const stack: MarkdownCarry = [{ kind: 'fence', opener: '```bash', closer: '```' }] + // closeMarks with a non-newline-terminated body will prepend \n + const result = closeMarks(stack) + expect(result).toBe('\n```') + }) + + it('closes multiple emphasis innermost-first', () => { + const stack: MarkdownCarry = [ + { kind: 'bold', opener: '**', closer: '**' }, + { kind: 'italic', opener: '*', closer: '*' }, + ] + // innermost = italic (index 1) β†’ close * first then ** + expect(closeMarks(stack)).toBe('***') + }) + + it('closes spoiler', () => { + const stack: MarkdownCarry = [{ kind: 'spoiler', opener: '||', closer: '||' }] + expect(closeMarks(stack)).toBe('||') + }) + + it('closes strike', () => { + const stack: MarkdownCarry = [{ kind: 'strike', opener: '~~', closer: '~~' }] + expect(closeMarks(stack)).toBe('~~') + }) +}) + +describe('reopenMarks', () => { + it('returns empty string for empty stack', () => { + expect(reopenMarks([])).toBe('') + }) + + it('reopens a fence with trailing newline', () => { + const stack: MarkdownCarry = [{ kind: 'fence', opener: '```python', closer: '```' }] + expect(reopenMarks(stack)).toBe('```python\n') + }) + + it('reopens multiple emphasis outermost-first', () => { + const stack: MarkdownCarry = [ + { kind: 'bold', opener: '**', closer: '**' }, + { kind: 'italic', opener: '*', closer: '*' }, + ] + // outermost = bold (index 0) β†’ reopen ** first then * + expect(reopenMarks(stack)).toBe('***') + }) + + it('reopens inline code without newline', () => { + const stack: MarkdownCarry = [{ kind: 'inlineCode', opener: '`', closer: '`' }] + expect(reopenMarks(stack)).toBe('`') + }) +}) + +describe('exclusiveOnly', () => { + it('keeps fence entries', () => { + const stack: MarkdownCarry = [{ kind: 'fence', opener: '```', closer: '```' }] + expect(exclusiveOnly(stack)).toHaveLength(1) + }) + + it('keeps inlineCode entries', () => { + const stack: MarkdownCarry = [{ kind: 'inlineCode', opener: '``', closer: '``' }] + expect(exclusiveOnly(stack)).toHaveLength(1) + }) + + it('removes all emphasis kinds', () => { + const stack: MarkdownCarry = [ + { kind: 'bold', opener: '**', closer: '**' }, + { kind: 'italic', opener: '*', closer: '*' }, + { kind: 'boldItalic', opener: '***', closer: '***' }, + { kind: 'underline', opener: '__', closer: '__' }, + { kind: 'strike', opener: '~~', closer: '~~' }, + { kind: 'spoiler', opener: '||', closer: '||' }, + ] + expect(exclusiveOnly(stack)).toHaveLength(0) + }) + + it('keeps fence but drops emphasis in mixed stack', () => { + const stack: MarkdownCarry = [ + { kind: 'fence', opener: '```', closer: '```' }, + { kind: 'bold', opener: '**', closer: '**' }, + ] + const result = exclusiveOnly(stack) + expect(result).toHaveLength(1) + expect(result[0]!.kind).toBe('fence') + }) +}) + +// --------------------------------------------------------------------------- +// 3. scanMarkdown +// --------------------------------------------------------------------------- + +describe('scanMarkdown', () => { + it('returns empty for plain text', () => { + expect(scanMarkdown('hello world')).toEqual([]) + }) + + it('returns empty for balanced fence', () => { + expect(scanMarkdown('```\ncode\n```')).toEqual([]) + }) + + it('detects unclosed fence (backtick)', () => { + const carry = scanMarkdown('```bash\necho hi') + expect(carry).toHaveLength(1) + expect(carry[0]!.kind).toBe('fence') + expect(carry[0]!.opener).toBe('```bash') + expect(carry[0]!.closer).toBe('```') + }) + + it('detects unclosed fence (tilde)', () => { + const carry = scanMarkdown('~~~python\nprint(1)') + expect(carry).toHaveLength(1) + expect(carry[0]!.kind).toBe('fence') + expect(carry[0]!.opener).toBe('~~~python') + expect(carry[0]!.closer).toBe('~~~') + }) + + it('detects unclosed 4-backtick fence', () => { + const carry = scanMarkdown('````ts\ncode') + expect(carry).toHaveLength(1) + expect(carry[0]!.opener).toBe('````ts') + expect(carry[0]!.closer).toBe('````') + }) + + it('detects unclosed 5-backtick fence', () => { + const carry = scanMarkdown('`````\ncode') + expect(carry[0]!.closer).toBe('`````') + }) + + it('detects unclosed inline code (single backtick)', () => { + const carry = scanMarkdown('hello `world') + expect(carry).toHaveLength(1) + expect(carry[0]!.kind).toBe('inlineCode') + expect(carry[0]!.opener).toBe('`') + }) + + it('detects unclosed inline code (double backtick)', () => { + const carry = scanMarkdown('test ``code') + expect(carry).toHaveLength(1) + expect(carry[0]!.kind).toBe('inlineCode') + expect(carry[0]!.opener).toBe('``') + }) + + it('closed inline code returns empty carry', () => { + expect(scanMarkdown('`hello`')).toEqual([]) + }) + + it('unmatched bold is in carry', () => { + const carry = scanMarkdown('this is **bold') + expect(carry.some((m) => m.kind === 'bold')).toBe(true) + }) + + it('matched bold returns empty carry', () => { + expect(scanMarkdown('**bold**')).toEqual([]) + }) + + it('unmatched italic is in carry', () => { + const carry = scanMarkdown('*italic without close') + expect(carry.some((m) => m.kind === 'italic')).toBe(true) + }) + + it('matched italic returns empty carry', () => { + expect(scanMarkdown('*italic*')).toEqual([]) + }) + + it('unmatched strike is in carry', () => { + const carry = scanMarkdown('~~strike without close') + expect(carry.some((m) => m.kind === 'strike')).toBe(true) + }) + + it('unmatched spoiler is in carry', () => { + const carry = scanMarkdown('||spoiler without close') + expect(carry.some((m) => m.kind === 'spoiler')).toBe(true) + }) + + it('backtick with backtick in info string is NOT a fence opener', () => { + // ```foo`bar β€” info contains backtick, invalid fence + const carry = scanMarkdown('```foo`bar\ncode') + // Should see inlineCode for ```, not a fence + // The ``` tries to open fence but info has backtick β†’ rejected + // Then ``` at line start with invalid info β†’ falls to inline code + // Let's just assert it doesn't open a fence + expect(carry.every((m) => m.kind !== 'fence')).toBe(true) + }) + + it('closing fence with info string is NOT a valid closer', () => { + // A closing fence with trailing content (not whitespace) does NOT close the fence + const carry = scanMarkdown('```\ncode\n``` not-a-close') + // The "``` not-a-close" line has non-whitespace after the marker β†’ not a valid close + expect(carry).toHaveLength(1) + expect(carry[0]!.kind).toBe('fence') + }) + + it('~~~ fence is not closed by ``` and vice versa', () => { + const carry = scanMarkdown('~~~\ncode\n```') + expect(carry).toHaveLength(1) + expect(carry[0]!.kind).toBe('fence') + expect(carry[0]!.opener).toBe('~~~') + }) + + it('``` inside ~~~ block is literal content, not a fence opener', () => { + const carry = scanMarkdown('~~~\n```\ncode\n~~~') + // ~~~ closes the ~~~ fence; the ``` inside is literal + expect(carry).toEqual([]) + }) + + it('scanMarkdown with startCarry', () => { + const startCarry = scanMarkdown('```python\nprint(1)') + expect(startCarry[0]!.kind).toBe('fence') + // Continue: provide the closing line + const carry2 = scanMarkdown('print(2)\n```\ndone', startCarry) + expect(carry2).toEqual([]) + }) + + it('emphasis inside fence is literal (no carry for it)', () => { + const carry = scanMarkdown('```\nrate **is** 5\n```') + expect(carry).toEqual([]) + }) + + it('emphasis inside unclosed fence is NOT in carry as emphasis', () => { + const carry = scanMarkdown('```\nrate **is** 5') + // Only the fence is open; ** is literal inside fence + expect(carry.every((m) => m.kind !== 'bold')).toBe(true) + }) + + it('intra-word underscore is NOT italic', () => { + const carry = scanMarkdown('word_other') + // The underscore is intra-word β†’ not italic + expect(carry).toEqual([]) + }) + + it('escaped asterisk is literal', () => { + const carry = scanMarkdown('hello \\* world') + expect(carry).toEqual([]) + }) +}) + +// --------------------------------------------------------------------------- +// 4. splitPreservingMarkdown β€” plain text / fast path +// --------------------------------------------------------------------------- + +describe('splitPreservingMarkdown β€” plain text', () => { + it('empty string β†’ single empty chunk', () => { + const r = splitPreservingMarkdown('', 100) + expect(r.chunks).toHaveLength(1) + expect(r.chunks[0]!.text).toBe('') + expect(r.endCarry).toEqual([]) + }) + + it('single char within limit β†’ single chunk', () => { + const r = splitPreservingMarkdown('x', 100) + expect(r.chunks).toEqual([{ text: 'x' }]) + }) + + it('text exactly maxLength β†’ single chunk (not two)', () => { + const text = 'a'.repeat(100) + const r = splitPreservingMarkdown(text, 100) + expect(r.chunks).toHaveLength(1) + expect(r.chunks[0]!.text).toBe(text) + }) + + it('text one over maxLength β†’ splits into two', () => { + const text = 'a'.repeat(101) + const r = splitPreservingMarkdown(text, 100) + expect(r.chunks.length).toBeGreaterThanOrEqual(2) + assertAllWithinLimit(r.chunks, 100) + expect(reconstruct(r.chunks)).toBe(text) + }) + + it('no-markdown fast path returns single chunk unchanged', () => { + const text = 'Hello, world! No markdown here.' + const r = splitPreservingMarkdown(text, 1000) + expect(r.chunks).toEqual([{ text }]) + expect(r.endCarry).toEqual([]) + }) + + it('multi-line plain text splits on newlines', () => { + const lines = Array.from({ length: 50 }, (_, i) => `line ${i} content`) + const text = lines.join('\n') + const r = splitPreservingMarkdown(text, 200) + assertAllWithinLimit(r.chunks, 200) + expect(reconstruct(r.chunks)).toBe(text) + }) + + it('endCarry is empty for fully plain text', () => { + const r = splitPreservingMarkdown('hello world, no markdown!', 100) + expect(r.endCarry).toEqual([]) + }) + + it('no bridge injected for all-plain-text split', () => { + const text = 'a'.repeat(50) + '\n' + 'b'.repeat(50) + const r = splitPreservingMarkdown(text, 60) + const anyBridge = r.chunks.some((c) => c.bridgeOpen || c.bridgeClose) + expect(anyBridge).toBe(false) + expect(reconstruct(r.chunks)).toBe(text) + }) +}) + +// --------------------------------------------------------------------------- +// 5. splitPreservingMarkdown β€” code fences +// --------------------------------------------------------------------------- + +describe('splitPreservingMarkdown β€” code fences', () => { + it('short fence fits in one chunk β†’ no split, no bridge', () => { + const text = '```bash\necho hello\n```' + const r = splitPreservingMarkdown(text, 200) + expect(r.chunks).toHaveLength(1) + expect(r.chunks[0]!.text).toBe(text) + expect(r.chunks[0]!.bridgeOpen).toBeUndefined() + expect(r.chunks[0]!.bridgeClose).toBeUndefined() + }) + + it('fence split in the middle β€” both halves are balanced (3 backticks)', () => { + const body = Array.from({ length: 40 }, (_, i) => `line ${i}`).join('\n') + const text = '```bash\n' + body + '\n```' + const r = splitPreservingMarkdown(text, 200) + expect(r.chunks.length).toBeGreaterThan(1) + for (const c of r.chunks) { + const count = countOccurrences(c.text, '```') + expect(count % 2).toBe(0) + } + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 200) + }) + + it('fence split β€” 4-backtick opener uses 4-backtick closer', () => { + const body = Array.from({ length: 30 }, (_, i) => `x${i}`).join('\n') + const text = '````ts\n' + body + '\n````' + const r = splitPreservingMarkdown(text, 150) + if (r.chunks.length > 1) { + for (const c of r.chunks) { + const count = countOccurrences(c.text, '````') + expect(count % 2).toBe(0) + } + } + expect(reconstruct(r.chunks)).toBe(text) + }) + + it('fence split β€” 5-backtick opener uses 5-backtick closer', () => { + const body = 'a\n'.repeat(30) + const text = '`````python\n' + body + '`````' + const r = splitPreservingMarkdown(text, 100) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 100) + }) + + it('fence split β€” tilde fence (~~~)', () => { + const body = Array.from({ length: 30 }, (_, i) => `step ${i}`).join('\n') + const text = '~~~\n' + body + '\n~~~' + const r = splitPreservingMarkdown(text, 150) + if (r.chunks.length > 1) { + for (const c of r.chunks) { + const count = countOccurrences(c.text, '~~~') + expect(count % 2).toBe(0) + } + } + expect(reconstruct(r.chunks)).toBe(text) + }) + + it('fence split β€” tilde fence with info string (~~~python)', () => { + const body = 'print(x)\n'.repeat(20) + const text = '~~~python\n' + body + '~~~' + const r = splitPreservingMarkdown(text, 100) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 100) + if (r.chunks.length > 1) { + expect(r.chunks[1]!.bridgeOpen).toBe('~~~python\n') + } + }) + + it('fence split β€” no info string', () => { + const body = 'code\n'.repeat(30) + const text = '```\n' + body + '```' + const r = splitPreservingMarkdown(text, 100) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 100) + }) + + it('backtick fence with backtick in info string is NOT a valid opener', () => { + // ```foo`bar is rejected as fence opener + const text = '```foo`bar\ncode line\n```' + const r = splitPreservingMarkdown(text, 500) + // There should be no fence carry since it's not a valid fence + expect(r.endCarry.every((m) => m.kind !== 'fence')).toBe(true) + }) + + it('closing fence with info string is NOT a valid closer β€” fence stays open', () => { + const text = '```\ncode\n``` extra' + const r = splitPreservingMarkdown(text, 500) + // The fence is still open (closer has non-whitespace trailing) + expect(r.endCarry.some((m) => m.kind === 'fence')).toBe(true) + }) + + it('~~~ fence NOT closed by ``` line', () => { + const text = '~~~\ncode\n```\nmore\n~~~' + const r = splitPreservingMarkdown(text, 500) + expect(r.endCarry).toEqual([]) + // The ``` inside ~~~ is content + }) + + it('``` inside ~~~ is literal content β€” reopener uses ~~~', () => { + const body = '```\ninner\n```\n'.repeat(10) + const text = '~~~\n' + body + '~~~' + const r = splitPreservingMarkdown(text, 80) + expect(reconstruct(r.chunks)).toBe(text) + // All chunks should use ~~~ for bridging, not ``` + for (const c of r.chunks) { + if (c.bridgeOpen) { + expect(c.bridgeOpen).toBe('~~~\n') + } + if (c.bridgeClose) { + expect(c.bridgeClose).toContain('~~~') + expect(c.bridgeClose).not.toContain('```') + } + } + }) + + it('unclosed fence β†’ endCarry reports it', () => { + const r = splitPreservingMarkdown('intro\n```js\nconst x = 1', 1000) + expect(r.endCarry.some((m) => m.kind === 'fence')).toBe(true) + }) + + it('unclosed fence β†’ exclusiveOnly has it', () => { + const r = splitPreservingMarkdown('intro\n```js\nconst x = 1', 1000) + expect(exclusiveOnly(r.endCarry)).toHaveLength(1) + }) + + it('fence split preserves opener in bridgeOpen of next chunk', () => { + const body = 'x\n'.repeat(30) + const text = '```bash\n' + body + '```' + const r = splitPreservingMarkdown(text, 100) + if (r.chunks.length > 1) { + expect(r.chunks[1]!.bridgeOpen).toBe('```bash\n') + } + }) + + it('fence split β€” bridgeClose ends with fence marker on its own line', () => { + const body = 'line\n'.repeat(30) + const text = '```\n' + body + '```' + const r = splitPreservingMarkdown(text, 100) + for (const c of r.chunks) { + if (c.bridgeClose) { + // bridgeClose for a fence should end with the marker + expect(c.bridgeClose.trimEnd().endsWith('```')).toBe(true) + } + } + }) + + it('two back-to-back code blocks in one chunk β€” no bridges injected', () => { + const text = '```js\na=1\n```\n```py\nb=2\n```' + const r = splitPreservingMarkdown(text, 200) + expect(r.chunks).toHaveLength(1) + expect(r.chunks[0]!.bridgeOpen).toBeUndefined() + expect(r.chunks[0]!.bridgeClose).toBeUndefined() + }) + + it('split exactly at the opener line boundary respects the limit', () => { + // Regression: the budget loop must size each chunk against the ACTUAL closer + // (closeMarksForBody, which skips a leading '\n' when the body already ends + // with one) rather than closeMarks (which always prepends '\n'). Otherwise a + // fence cut at a newline boundary overflows maxLength by 1. + const body = 'code\n'.repeat(3) + const text = '```\n' + body + '```' + const r = splitPreservingMarkdown(text, 9) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 9) + }) + + it('all-fence content (no prose) round-trips', () => { + const text = '```\n' + 'line\n'.repeat(50) + '```' + const r = splitPreservingMarkdown(text, 100) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 100) + }) + + it('split with text after the fence', () => { + const body = 'code\n'.repeat(20) + const text = '```bash\n' + body + '```\nAfterText here' + const r = splitPreservingMarkdown(text, 100) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 100) + // Last chunk (or the one after fence) should not be fenced + const lastChunk = r.chunks[r.chunks.length - 1]! + expect(lastChunk.text).toContain('AfterText') + expect(r.endCarry).toEqual([]) + }) + + it('split with text before and after the fence', () => { + const prose = 'Before text\n' + const body = 'code\n'.repeat(20) + const text = prose + '```\n' + body + '```\nAfter text' + const r = splitPreservingMarkdown(text, 100) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 100) + }) + + it('4-tilde fence (~~~~) uses 4-tilde closer', () => { + const carry = scanMarkdown('~~~~\ncode') + expect(carry[0]!.opener).toBe('~~~~') + expect(carry[0]!.closer).toBe('~~~~') + }) + + it('fence with no newline at body end still closes properly', () => { + const text = '```\ncode without final newline```' + // The closing ``` is NOT on its own line here, so fence stays open + const r = splitPreservingMarkdown(text, 500) + // fence is unclosed because ``` is mid-line + expect(r.endCarry.some((m) => m.kind === 'fence')).toBe(true) + }) +}) + +// --------------------------------------------------------------------------- +// 6. splitPreservingMarkdown β€” inline code +// --------------------------------------------------------------------------- + +describe('splitPreservingMarkdown β€” inline code', () => { + it('short inline code fits in one chunk β€” no split', () => { + const text = 'Use `console.log()` here' + const r = splitPreservingMarkdown(text, 200) + expect(r.chunks).toHaveLength(1) + }) + + it('unclosed inline code β†’ endCarry', () => { + const r = splitPreservingMarkdown('hello `world', 1000) + expect(r.endCarry.some((m) => m.kind === 'inlineCode')).toBe(true) + }) + + it('closed inline code β†’ no endCarry', () => { + const r = splitPreservingMarkdown('`hello`', 1000) + expect(r.endCarry).toEqual([]) + }) + + it('double-backtick inline code β€” run-length matched close', () => { + const text = '``code with `backtick` inside``' + const r = splitPreservingMarkdown(text, 200) + expect(r.endCarry).toEqual([]) + expect(r.chunks).toHaveLength(1) + }) + + it('inline code does not match different run length', () => { + // `` opened, closed by single ` β€” should NOT close it + const text = '``code`rest' + const r = splitPreservingMarkdown(text, 200) + // The `` is still open (single ` inside doesn't close it) + expect(r.endCarry.some((m) => m.kind === 'inlineCode')).toBe(true) + }) + + it('inline code split across boundary β€” each piece is self-contained', () => { + const inline = '`' + 'x'.repeat(80) + '`' + const text = 'before ' + inline + ' after' + const r = splitPreservingMarkdown(text, 50) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 50) + for (const c of r.chunks) { + const count = countOccurrences(c.text, '`') + expect(count % 2).toBe(0) + } + }) + + it('exclusiveOnly includes inlineCode carry', () => { + const carry = scanMarkdown('hello `world') + expect(exclusiveOnly(carry)).toHaveLength(1) + expect(exclusiveOnly(carry)[0]!.kind).toBe('inlineCode') + }) + + it('emphasis inside inline code is literal (not parsed)', () => { + const carry = scanMarkdown('`**not bold**') + // inlineCode is open; ** inside is literal + expect(carry).toHaveLength(1) + expect(carry[0]!.kind).toBe('inlineCode') + }) + + it('inline code in sentence β€” round-trips on split', () => { + const word = '`' + 'v'.repeat(30) + '`' + const text = 'a '.repeat(30) + word + ' b'.repeat(30) + const r = splitPreservingMarkdown(text, 80) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 80) + }) +}) + +// --------------------------------------------------------------------------- +// 7. splitPreservingMarkdown β€” emphasis +// --------------------------------------------------------------------------- + +describe('splitPreservingMarkdown β€” emphasis', () => { + it('matched bold β†’ no endCarry', () => { + const r = splitPreservingMarkdown('**bold**', 1000) + expect(r.endCarry).toEqual([]) + }) + + it('unmatched bold β†’ endCarry (emphasis kind)', () => { + const carry = scanMarkdown('**bold without close') + expect(carry.some((m) => m.kind === 'bold')).toBe(true) + }) + + it('unmatched bold NOT in exclusiveOnly', () => { + const carry = scanMarkdown('**bold') + expect(exclusiveOnly(carry)).toHaveLength(0) + }) + + it('bold split β€” each chunk has even number of **', () => { + // Use join(' ') so the closing ** is not preceded by a space + // (a space before ** makes it non-right-flanking β†’ not a valid closer) + const text = '**' + Array.from({ length: 60 }, (_, i) => `word${i}`).join(' ') + '**' + const r = splitPreservingMarkdown(text, 120) + expect(r.chunks.length).toBeGreaterThan(1) + for (const c of r.chunks) { + const n = countOccurrences(c.text, '**') + expect(n % 2).toBe(0) + } + expect(reconstruct(r.chunks)).toBe(text) + }) + + it('italic (*) split β€” bridges correctly in each chunk', () => { + // NOTE: closing * must NOT be preceded by whitespace (would make it non-right-flanking) + // Use join(' ') to avoid trailing space before closing * + const text = '*' + Array.from({ length: 60 }, (_, i) => `word${i}`).join(' ') + '*' + const r = splitPreservingMarkdown(text, 100) + expect(r.chunks.length).toBeGreaterThan(1) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 100) + // Each chunk should have balanced * (from bridge) + for (const c of r.chunks) { + const n = countOccurrences(c.text, '*') + expect(n % 2).toBe(0) + } + }) + + it('italic (_) split β€” bridges correctly in each chunk', () => { + // Use join(' ') to avoid trailing space before closing _ (which blocks closing) + const text = '_' + Array.from({ length: 60 }, (_, i) => `word${i}`).join(' ') + '_' + const r = splitPreservingMarkdown(text, 100) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 100) + if (r.chunks.length > 1) { + for (const c of r.chunks) { + const n = countOccurrences(c.text, '_') + expect(n % 2).toBe(0) + } + } + }) + + it('underline (__) split β€” bridges correctly in each chunk', () => { + // Use join(' ') to avoid trailing space before closing __ (which blocks closing) + const text = '__' + Array.from({ length: 60 }, (_, i) => `word${i}`).join(' ') + '__' + const r = splitPreservingMarkdown(text, 100) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 100) + if (r.chunks.length > 1) { + for (const c of r.chunks) { + const n = countOccurrences(c.text, '__') + expect(n % 2).toBe(0) + } + } + }) + + it('boldItalic (***) split β€” balanced', () => { + // Use join(' ') to avoid trailing space before closing *** + const text = '***' + Array.from({ length: 60 }, (_, i) => `word${i}`).join(' ') + '***' + const r = splitPreservingMarkdown(text, 100) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 100) + }) + + it('strikethrough (~~) split β€” balanced', () => { + // Use join(' ') so closing ~~ is not preceded by whitespace (which blocks closing) + const text = '~~' + Array.from({ length: 60 }, (_, i) => `word${i}`).join(' ') + '~~' + const r = splitPreservingMarkdown(text, 100) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 100) + if (r.chunks.length > 1) { + for (const c of r.chunks) { + const n = countOccurrences(c.text, '~~') + expect(n % 2).toBe(0) + } + } + }) + + it('spoiler (||) split β€” balanced', () => { + // Use join(' ') to avoid trailing space before closing || + const text = '||' + Array.from({ length: 60 }, (_, i) => `word${i}`).join(' ') + '||' + const r = splitPreservingMarkdown(text, 100) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 100) + if (r.chunks.length > 1) { + for (const c of r.chunks) { + const n = countOccurrences(c.text, '||') + expect(n % 2).toBe(0) + } + } + }) + + it('unmatched lone asterisk (math) β€” no bridge injected', () => { + const text = 'a'.repeat(100) + ' 2 * 3 is math' + const r = splitPreservingMarkdown(text, 50) + expect(reconstruct(r.chunks)).toBe(text) + const anyBridge = r.chunks.some((c) => c.bridgeOpen || c.bridgeClose) + expect(anyBridge).toBe(false) + }) + + it('right-flanking ** with whitespace before β€” NOT a valid closer', () => { + // "** close" β€” the ** has a space before it (after), so it is NOT right-flanking + // The ** before is left-flanking (nothing before), this ** has space before β†’ not a match + // Actually: "bold **" trailing space means ** may not be a closer + // The text: **bold ** β€” the closer "**" has a space immediately before it (word char?) + // Let's test a clear case: opening with no close + const carry = scanMarkdown('**bold **') + // " **" β€” two spaces before **, so before = ' ' which is whitespace β†’ not right-flanking β†’ not a closer + // So **bold ** should NOT close the bold + // (checking the implementation behavior) + // The opening ** sees: before=undefined (start), after='b' β†’ canOpen=true + // The closing ** sees: before=' ' β†’ canClose = !isWhitespace(' ') = false β†’ not a closer + expect(carry.some((m) => m.kind === 'bold')).toBe(true) + }) + + it('intra-word underscore is NOT italic', () => { + const carry = scanMarkdown('word_other_word') + expect(carry).toEqual([]) + }) + + it('escaped asterisk is literal β€” no carry', () => { + const carry = scanMarkdown('hello \\* world \\*') + expect(carry).toEqual([]) + }) + + it('escaped backtick is literal', () => { + const carry = scanMarkdown('\\`not code\\`') + expect(carry).toEqual([]) + }) + + it('lone ~ (not ~~) is literal', () => { + const carry = scanMarkdown('~single tilde~') + // ~ with len=1 has no kind in EMPHASIS_BY_CHAR β†’ literal + expect(carry).toEqual([]) + }) + + it('lone | is literal', () => { + const carry = scanMarkdown('|single pipe|') + // | with len=1 has no kind β†’ literal + expect(carry).toEqual([]) + }) + + it('bridgeOpen for bold split is **', () => { + // Use join(' ') so closing ** is not preceded by a space (which blocks closing) + const text = '**' + Array.from({ length: 60 }, (_, i) => `word${i}`).join(' ') + '**' + const r = splitPreservingMarkdown(text, 120) + expect(r.chunks.length).toBeGreaterThan(1) + expect(r.chunks[1]!.bridgeOpen).toBe('**') + }) + + it('bridgeClose for bold split is **', () => { + // Use join(' ') so closing ** is not preceded by a space (which blocks closing) + const text = '**' + Array.from({ length: 60 }, (_, i) => `word${i}`).join(' ') + '**' + const r = splitPreservingMarkdown(text, 120) + expect(r.chunks.length).toBeGreaterThan(1) + expect(r.chunks[0]!.bridgeClose).toBe('**') + }) + + it('italic inside bold split β€” bridgeClose closes italic first', () => { + // **bold *italic* back** + // If split inside *italic*, close * before ** + const text = '**bold ' + '*italic '.repeat(10) + '* back**' + const r = splitPreservingMarkdown(text, 40) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 40) + }) +}) + +// --------------------------------------------------------------------------- +// 8. Nesting & ordering of close/reopen +// --------------------------------------------------------------------------- + +describe('splitPreservingMarkdown β€” nesting and ordering', () => { + it('nested bold+italic: close inner first, reopen outer first', () => { + // **bold _italic_** β€” if split mid-italic, bridgeClose should be "_**" and bridgeOpen should be "**_" + const text = '**bold _' + 'w '.repeat(30) + '_back**' + const r = splitPreservingMarkdown(text, 60) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 60) + // Check ordering on a chunk that has both open + for (const c of r.chunks) { + if (c.bridgeClose && c.bridgeOpen) { + // bridgeClose: innermost first (italic before bold) + const closeIdx_italic = c.bridgeClose.indexOf('_') + const closeIdx_bold = c.bridgeClose.indexOf('**') + if (closeIdx_italic !== -1 && closeIdx_bold !== -1) { + expect(closeIdx_italic).toBeLessThan(closeIdx_bold) + } + // bridgeOpen: outermost first (bold before italic) + const openIdx_bold = c.bridgeOpen.indexOf('**') + const openIdx_italic = c.bridgeOpen.indexOf('_') + if (openIdx_bold !== -1 && openIdx_italic !== -1) { + expect(openIdx_bold).toBeLessThan(openIdx_italic) + } + } + } + }) + + it('bold inside italic: round-trips', () => { + const text = '_italic **bold** end_' + const r = splitPreservingMarkdown(text, 15) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 15) + }) + + it('emphasis inside code fence is not bridged as emphasis', () => { + const body = '**not bridged**\n'.repeat(10) + const text = '```\n' + body + '```' + const r = splitPreservingMarkdown(text, 80) + expect(reconstruct(r.chunks)).toBe(text) + for (const c of r.chunks) { + // No emphasis-type bridge β€” only fence bridges allowed + if (c.bridgeOpen) expect(c.bridgeOpen).toMatch(/^(`{3,}|~{3,})/) + if (c.bridgeClose) expect(c.bridgeClose.trim()).toMatch(/^(`{3,}|~{3,})$/) + } + }) + + it('multiple adjacent bold spans β€” only open ones bridged', () => { + const text = '**one** and **' + 'long '.repeat(30) + '**' + const r = splitPreservingMarkdown(text, 80) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 80) + }) +}) + +// --------------------------------------------------------------------------- +// 9. startCarry continuation +// --------------------------------------------------------------------------- + +describe('splitPreservingMarkdown β€” startCarry', () => { + it('inherited fence carry β€” first chunk prepends reopener', () => { + const carry = scanMarkdown('```python\nprint(1)') + const r = splitPreservingMarkdown('print(2)\n```\ndone', 1750, carry) + expect(r.chunks[0]!.text.startsWith('```python\n')).toBe(true) + expect(r.chunks[0]!.bridgeOpen).toBe('```python\n') + }) + + it('inherited fence carry β€” round-trip', () => { + const carry = scanMarkdown('```python\nprint(1)') + const text = 'print(2)\n```\ndone' + const r = splitPreservingMarkdown(text, 1750, carry) + expect(reconstruct(r.chunks)).toBe(text) + }) + + it('inherited fence carry β€” each chunk within limit', () => { + const carry = scanMarkdown('```bash\necho start') + const body = 'line\n'.repeat(40) + const text = body + '```\npost' + const r = splitPreservingMarkdown(text, 100, carry) + assertAllWithinLimit(r.chunks, 100) + expect(reconstruct(r.chunks)).toBe(text) + }) + + it('inherited fence carry β€” budget accounts for reopener length', () => { + // reopener is "```bash\n" (8 chars); with maxLength=50, first chunk body ≀ 42 + const carry = scanMarkdown('```bash\n') + const text = 'x'.repeat(100) + const r = splitPreservingMarkdown(text, 50, carry) + assertAllWithinLimit(r.chunks, 50) + expect(reconstruct(r.chunks)).toBe(text) + }) + + it('inherited bold carry β€” first chunk prepends **', () => { + const carry: MarkdownCarry = [{ kind: 'bold', opener: '**', closer: '**' }] + const text = 'continued bold text here' + const r = splitPreservingMarkdown(text, 1000, carry) + expect(r.chunks[0]!.bridgeOpen).toBe('**') + expect(r.chunks[0]!.text.startsWith('**')).toBe(true) + }) + + it('inherited fence carry that is closed mid-text β†’ no endCarry fence', () => { + const carry = scanMarkdown('```\ncode') + const r = splitPreservingMarkdown('more\n```\nplain', 1000, carry) + expect(r.endCarry.every((m) => m.kind !== 'fence')).toBe(true) + }) + + it('carries from scanMarkdown with a tilde fence', () => { + const carry = scanMarkdown('~~~bash\nstart') + expect(carry[0]!.opener).toBe('~~~bash') + const r = splitPreservingMarkdown('middle\n~~~\nend', 1000, carry) + expect(r.chunks[0]!.bridgeOpen).toBe('~~~bash\n') + expect(reconstruct(r.chunks)).toBe('middle\n~~~\nend') + }) + + it('empty startCarry behaves like no carry', () => { + const r1 = splitPreservingMarkdown('hello', 100, []) + const r2 = splitPreservingMarkdown('hello', 100) + expect(r1.chunks).toEqual(r2.chunks) + expect(r1.endCarry).toEqual(r2.endCarry) + }) +}) + +// --------------------------------------------------------------------------- +// 10. Edge cases +// --------------------------------------------------------------------------- + +describe('splitPreservingMarkdown β€” edge cases', () => { + it('text is only newlines', () => { + const text = '\n\n\n\n\n' + const r = splitPreservingMarkdown(text, 10) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 10) + }) + + it('maxLength of 4 β€” still produces chunks without stalling', () => { + // Very small maxLength to test the guard-against-stall path + const text = 'abcde' + const r = splitPreservingMarkdown(text, 4) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 4) + }) + + it('large all-fence text with moderate maxLength', () => { + const body = 'line\n'.repeat(100) + const text = '```\n' + body + '```' + const r = splitPreservingMarkdown(text, 80) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 80) + }) + + it('no markdown fast path: hasNoDelimiters β†’ single chunk', () => { + // Text with no backtick, asterisk, underscore, tilde, pipe, or backslash + const text = 'Hello world! This is a plain sentence. 12345 abc DEF.' + const r = splitPreservingMarkdown(text, 1000) + expect(r.chunks).toHaveLength(1) + expect(r.chunks[0]!.text).toBe(text) + }) + + it('text with only backslash β€” delimiter detected, no fast path, but no carry', () => { + const text = 'hello \\ world' + const r = splitPreservingMarkdown(text, 1000) + expect(reconstruct(r.chunks)).toBe(text) + expect(r.endCarry).toEqual([]) + }) + + it('chunks array is never empty', () => { + const r = splitPreservingMarkdown('', 100) + expect(r.chunks.length).toBeGreaterThan(0) + }) + + it('a single very long line inside a fence β€” hard-splits but each piece is fenced', () => { + const longLine = 'x'.repeat(100) + const text = '```\n' + longLine + '\n```' + // maxLength=40: overhead is "```\n" (4) + "\n```" (4) = 8, leaving 32 chars per chunk + const r = splitPreservingMarkdown(text, 40) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 40) + for (const c of r.chunks) { + const count = countOccurrences(c.text, '```') + expect(count % 2).toBe(0) + } + }) + + it('text that ends immediately after fence opener', () => { + const text = '```' + const r = splitPreservingMarkdown(text, 1000) + expect(reconstruct(r.chunks)).toBe(text) + }) + + it('fence opener on last line (no newline after)', () => { + const text = 'prose\n```' + const r = splitPreservingMarkdown(text, 1000) + expect(reconstruct(r.chunks)).toBe(text) + }) + + it('mix of ``` and ~~~ fences (different types)', () => { + const text = '```\ncode1\n```\n~~~\ncode2\n~~~' + const r = splitPreservingMarkdown(text, 500) + expect(r.endCarry).toEqual([]) + expect(reconstruct(r.chunks)).toBe(text) + }) + + it('text with all emphasis kinds back to back respects the limit', () => { + // Regression: when avoidDelimiterBisect pulls `cut` back to a construct's + // contentEnd it adds that construct's closer to the stack; the budget loop + // re-derives the close-stack at the FINAL cut so those closer chars are + // always counted and no chunk exceeds maxLength. + const text = '**bold** _italic_ __under__ ~~strike~~ ||spoiler|| ***bi***' + const r = splitPreservingMarkdown(text, 20) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 20) + }) + + it('endCarry is empty after fully matched text', () => { + const text = '**bold** `code` ~~strike~~' + const r = splitPreservingMarkdown(text, 1000) + expect(r.endCarry).toEqual([]) + }) + + it('startCarry + text requires no split β†’ single chunk with bridgeOpen only', () => { + const carry: MarkdownCarry = [{ kind: 'bold', opener: '**', closer: '**' }] + const text = 'short' + const r = splitPreservingMarkdown(text, 1000, carry) + expect(r.chunks).toHaveLength(1) + expect(r.chunks[0]!.bridgeOpen).toBe('**') + }) + + it('fence with 1-3 spaces of indent is still a valid fence opener', () => { + const text = ' ```\ncode\n```' + const r = splitPreservingMarkdown(text, 1000) + expect(r.endCarry).toEqual([]) // closed + }) + + it('fence with 4 spaces of indent is NOT a valid fence opener', () => { + // 4 spaces = code block, not fence + const text = ' ```\ncode\n```' + const r = splitPreservingMarkdown(text, 1000) + // The ``` with 4 spaces prefix is not a fence opener in CommonMark + // The last ``` without indent is not a fence closer (no open fence) + expect(r.endCarry.every((m) => m.kind !== 'fence')).toBe(true) + }) + + it('multiple splits β€” all chunks within limit, round-trips', () => { + const text = Array.from({ length: 100 }, (_, i) => `line ${i}: ${'content '.repeat(5)}`).join('\n') + const r = splitPreservingMarkdown(text, 150) + assertAllWithinLimit(r.chunks, 150) + expect(reconstruct(r.chunks)).toBe(text) + }) + + it('inline code bridging β€” bridgeOpen and bridgeClose are the opener', () => { + const inline = '`' + 'x'.repeat(200) + '`' + const r = splitPreservingMarkdown(inline, 80) + if (r.chunks.length > 1) { + expect(r.chunks[1]!.bridgeOpen).toBe('`') + expect(r.chunks[0]!.bridgeClose).toBe('`') + } + }) + + it('text with only a single delimiter char β€” no crash', () => { + for (const ch of ['`', '*', '_', '~', '|', '\\']) { + const r = splitPreservingMarkdown(ch, 100) + expect(r.chunks.length).toBeGreaterThan(0) + expect(reconstruct(r.chunks)).toBe(ch) + } + }) +}) + +// --------------------------------------------------------------------------- +// 11. Invariant stress tests +// --------------------------------------------------------------------------- + +describe('splitPreservingMarkdown β€” stress / invariants', () => { + it('invariant: every chunk <= maxLength for mixed content', () => { + const text = [ + 'Intro paragraph.', + '```bash', + 'echo hello', + 'echo world', + '```', + '**Bold text** and _italic_ and ~~strike~~ and ||spoiler||.', + '```js', + "const x = 'hello'", + '```', + 'Conclusion.', + ].join('\n') + const maxLength = 60 + const r = splitPreservingMarkdown(text, maxLength) + assertAllWithinLimit(r.chunks, maxLength) + expect(reconstruct(r.chunks)).toBe(text) + }) + + it('invariant: round-trip for repeated emphasis splits', () => { + const text = '**' + ('word '.repeat(20) + '\n').repeat(5) + '**' + const r = splitPreservingMarkdown(text, 80) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 80) + }) + + it('invariant: round-trip for fence inside prose (both sides)', () => { + const before = 'This is some prose before. ' + const body = 'code_line\n'.repeat(20) + const after = '\nAnd some prose after.' + const text = before + '```python\n' + body + '```' + after + const r = splitPreservingMarkdown(text, 100) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, 100) + }) + + it('invariant: chunks.length >= 1 always', () => { + for (const text of ['', 'a', '**b**', '```\ncode\n```', '`x`']) { + const r = splitPreservingMarkdown(text, 50) + expect(r.chunks.length).toBeGreaterThanOrEqual(1) + } + }) + + it('invariant: reconstruct works for multi-split fence', () => { + const body = 'item\n'.repeat(100) + const text = '```\n' + body + '```' + for (const maxLen of [50, 100, 200]) { + const r = splitPreservingMarkdown(text, maxLen) + expect(reconstruct(r.chunks)).toBe(text) + assertAllWithinLimit(r.chunks, maxLen) + } + }) +}) + +// --------------------------------------------------------------------------- +// 11. Cross-send continuity (the reported bug) +// --------------------------------------------------------------------------- +// A fence opened before a tool call and continued after it spans two +// independent sends (a streaming pre-tool flush, then the final send). +// AgentLoop.sendSegments threads endCarry β†’ startCarry across the two; this +// models that threading at the utility level. +describe('splitPreservingMarkdown β€” cross-send continuity', () => { + it('keeps the fence balanced across two separate sends', () => { + const preTool = 'Here are the flags:\n```bash\n--disable-blink-features=AutomationControlled' + const postTool = '--password-store=basic\n```\nThat avoids keyring weirdness.' + + const r1 = splitPreservingMarkdown(preTool, 1750, []) + const carry = exclusiveOnly(r1.endCarry) // only exclusive constructs cross a send boundary + expect(carry.map((m) => m.kind)).toEqual(['fence']) + const r2 = splitPreservingMarkdown(postTool, 1750, carry) + + const all = [...r1.chunks, ...r2.chunks] + // Every Discord message renders with a balanced fence count. + for (const c of all) { + expect((c.text.match(/```/g) || []).length % 2).toBe(0) + } + // The second send reopens the fence the first one closed. + expect(r2.chunks[0]!.text.startsWith('```bash\n')).toBe(true) + expect(r2.chunks[0]!.bridgeOpen).toBe('```bash\n') + // Context reconstruction strips the synthetic bridges back to the original. + expect(reconstruct(all)).toBe(preTool + postTool) + }) + + it('drops dangling emphasis at a send boundary (not carried)', () => { + const r = splitPreservingMarkdown('this is **bold', 1750, []) + expect(r.endCarry.map((m) => m.kind)).toEqual(['bold']) + expect(exclusiveOnly(r.endCarry)).toEqual([]) // emphasis does not cross a send + }) +}) + +// --------------------------------------------------------------------------- +// 12. Regression: fuzz-found defects +// --------------------------------------------------------------------------- + +/** Run-merging delimiter chars (backslash escapes, so it is excluded). */ +const SEAM_CHARS = new Set(['`', '~', '*', '_', '|']) + +/** Strip the bridges from a chunk to recover its original body slice. */ +function bodyOf(c: ChunkPiece): string { + let t = c.text + if (c.bridgeOpen && t.startsWith(c.bridgeOpen)) t = t.slice(c.bridgeOpen.length) + if (c.bridgeClose && t.endsWith(c.bridgeClose)) t = t.slice(0, t.length - c.bridgeClose.length) + return t +} + +/** + * Assert no chunk fuses a synthetic bridge with an identical delimiter run in + * the adjacent body (defect 2): the reopener's last char must differ from the + * body's first char, and the closer's first char from the body's last char. + */ +function assertNoSeamMerge(chunks: ChunkPiece[]): void { + for (const c of chunks) { + const body = bodyOf(c) + if (!body.length) continue + if (c.bridgeOpen) { + const last = c.bridgeOpen[c.bridgeOpen.length - 1]! + if (SEAM_CHARS.has(last)) expect(last).not.toBe(body[0]) + } + if (c.bridgeClose) { + const first = c.bridgeClose[0]! + if (SEAM_CHARS.has(first)) expect(first).not.toBe(body[body.length - 1]) + } + } +} + +describe('splitPreservingMarkdown β€” defect 1: giant fence reopener', () => { + it('does not exceed maxLength when a fence with a long opening line straddles a cut', () => { + const text = + '~~~tyttykittykittykittykittyboundary ipsum_kittyprobe ____***ζ—₯本θͺž lorem semicoloony' + + 'kitty'.repeat(43) + + 'kit|kitty the**bold**the _x |||ipsum probe \\*~the*i*naΓ―ve kitty lorem sem\nn' + const r = splitPreservingMarkdown(text, 300) + assertAllWithinLimit(r.chunks, 300) + expect(reconstruct(r.chunks)).toBe(text) + }) + + it('reopens a long-info fence with just the marker + first info word (not the whole line)', () => { + const opener = '```' + 'averylongfirstword'.repeat(6) + ' rest of the info string here' + const body = 'code line\n'.repeat(40) + const text = opener + '\n' + body + '```' + const r = splitPreservingMarkdown(text, 120) + assertAllWithinLimit(r.chunks, 120) + expect(reconstruct(r.chunks)).toBe(text) + // Any fence reopener is the marker + a single info word, never the verbatim line. + for (const c of r.chunks) { + if (c.bridgeOpen && c.bridgeOpen.startsWith('```')) { + expect(c.bridgeOpen).not.toContain(' ') + expect(c.bridgeOpen.length).toBeLessThan(opener.length) + } + } + }) + + it('hard-guarantees invariant A across small maxLengths for a pathological fence', () => { + const text = '~~~' + 'infoword'.repeat(30) + '\n' + 'x'.repeat(400) + '\ntail' + for (const maxLength of [60, 80, 120, 300]) { + const r = splitPreservingMarkdown(text, maxLength) + assertAllWithinLimit(r.chunks, maxLength) + expect(reconstruct(r.chunks)).toBe(text) + } + }) +}) + +describe('splitPreservingMarkdown β€” defect 2: bridge/body delimiter-run merge at the seam', () => { + it('(a) reopened inline code does not fuse with a leading body backtick', () => { + const text = 'he a `\n`x' + 'kitty'.repeat(14) + 'k ' + const r = splitPreservingMarkdown(text, 80) + assertAllWithinLimit(r.chunks, 80) + expect(reconstruct(r.chunks)).toBe(text) + assertNoSeamMerge(r.chunks) + // No chunk starts a spurious double-backtick run from bridge + body. + for (const c of r.chunks) expect(c.text).not.toContain('``x') + }) + + it('(b) strike closer does not fuse with a trailing opener to form ~~~~', () => { + const text = 'x ipsum ' + 'kitty'.repeat(5) + '**a~~s~~' + const r = splitPreservingMarkdown(text, 40) + assertAllWithinLimit(r.chunks, 40) + expect(reconstruct(r.chunks)).toBe(text) + assertNoSeamMerge(r.chunks) + for (const c of r.chunks) expect(c.text).not.toContain('~~~~') + }) + + it('(c) backtick runs do not merge on either side of a seam', () => { + const text = 'um `code`_~~~````\n`_lm' + 'lorem'.repeat(60) + const r = splitPreservingMarkdown(text, 200) + assertAllWithinLimit(r.chunks, 200) + expect(reconstruct(r.chunks)).toBe(text) + assertNoSeamMerge(r.chunks) + }) + + it('final chunk does not fuse an inline-code closer with trailing body backticks', () => { + // Unclosed inline code whose original content ends in backticks: Discord + // already renders it literally, so the final chunk leaves it open rather + // than emitting a closer that would fuse into a longer backtick run. + const text = '**bold**``````the lorem ' + 'ipsum'.repeat(50) + '**bold**```' + const r = splitPreservingMarkdown(text, 2000) + expect(reconstruct(r.chunks)).toBe(text) + assertNoSeamMerge(r.chunks) + }) +}) diff --git a/portal-relay/test/split-send.test.ts b/portal-relay/test/split-send.test.ts new file mode 100644 index 0000000..e426bf9 --- /dev/null +++ b/portal-relay/test/split-send.test.ts @@ -0,0 +1,149 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { splitPreservingMarkdown } from '../src/discord-markdown.js'; +import { MessageStore } from '../src/message-store.js'; +import { stripBridges } from '../src/relay.js'; +import { PartialSendError, WebhookPool, type WebhookOps, type WebhookSendOpts } from '../src/webhook-pool.js'; + +const tmpFile = () => join(mkdtempSync(join(tmpdir(), 'portal-split-')), 'attr.json'); + +// ── WebhookPool.sendMany ── + +function fakeOps(failAt?: number): { ops: WebhookOps; sent: WebhookSendOpts[] } { + const sent: WebhookSendOpts[] = []; + let n = 0; + const ops: WebhookOps = { + ensureWebhooks: async () => ['wh1'], + sendWebhook: async (_id, opts) => { + if (failAt !== undefined && n === failAt) throw new Error('boom'); + sent.push(opts); + return { messageId: `m${++n}` }; + }, + editWebhookMessage: async () => {}, + deleteWebhookMessage: async () => {}, + }; + return { ops, sent }; +} + +test('sendMany sends all parts in order on one webhook', async () => { + const { ops, sent } = fakeOps(); + const pool = new WebhookPool(ops, 1); + const base = { username: 'u', avatarURL: '' }; + const res = await pool.sendMany('chan', 'p1', [ + { ...base, content: 'one' }, + { ...base, content: 'two' }, + { ...base, content: 'three' }, + ]); + assert.deepEqual(res.messageIds, ['m1', 'm2', 'm3']); + assert.deepEqual(sent.map((s) => s.content), ['one', 'two', 'three']); +}); + +test('sendMany keeps parts contiguous against concurrent sends', async () => { + const { ops, sent } = fakeOps(); + const pool = new WebhookPool(ops, 1); + const base = { username: 'u', avatarURL: '' }; + const [multi] = await Promise.all([ + pool.sendMany('chan', 'p1', [ + { ...base, content: 'a1' }, + { ...base, content: 'a2' }, + ]), + pool.send('chan', 'p1', { ...base, content: 'b' }), + ]); + assert.equal(multi.messageIds.length, 2); + const order = sent.map((s) => s.content); + const i1 = order.indexOf('a1'); + // The two parts must be adjacent β€” 'b' cannot land between them. + assert.equal(order[i1 + 1], 'a2'); +}); + +test('sendMany surfaces a PartialSendError carrying the ids that DID send', async () => { + const { ops } = fakeOps(1); // second send fails + const pool = new WebhookPool(ops, 1); + const base = { username: 'u', avatarURL: '' }; + await assert.rejects( + pool.sendMany('chan', 'p1', [ + { ...base, content: 'one' }, + { ...base, content: 'two' }, + ]), + (err: unknown) => { + assert.ok(err instanceof PartialSendError); + assert.deepEqual(err.sentIds, ['m1']); + return true; + }, + ); +}); + +// ── MessageStore split metadata ── + +test('split metadata round-trips through attribution persistence', () => { + const path = tmpFile(); + const s1 = new MessageStore({ path }); + s1.record({ + channelId: 'c', guildId: 'g', discordMsgId: 'd1', personaId: 'p', webhookId: 'w', + bridgeClose: '```', parts: ['d1', 'd2'], + }); + s1.record({ + channelId: 'c', guildId: 'g', discordMsgId: 'd2', personaId: 'p', webhookId: 'w', + bridgeOpen: '```ts\n', partOf: 'd1', + }); + s1.flush(); + + const s2 = new MessageStore({ path }); + const first = s2.getByDiscordId('d1')!; + const second = s2.getByDiscordId('d2')!; + assert.deepEqual(first.parts, ['d1', 'd2']); + assert.equal(first.bridgeClose, '```'); + assert.equal(second.partOf, 'd1'); + assert.equal(second.bridgeOpen, '```ts\n'); +}); + +test('setSplitMeta overwrites bridges β€” including back to undefined', () => { + const s = new MessageStore(); + s.record({ + channelId: 'c', guildId: 'g', discordMsgId: 'd1', personaId: 'p', + bridgeClose: '**', parts: ['d1', 'd2'], + }); + s.setSplitMeta('d1', {}); + const ref = s.getByDiscordId('d1')!; + assert.equal(ref.bridgeClose, undefined); + assert.equal(ref.parts, undefined); +}); + +// ── stripBridges ── + +test('stripBridges removes exact reopener and closer', () => { + assert.equal(stripBridges('```ts\ncode()', { bridgeOpen: '```ts\n' }), 'code()'); + assert.equal(stripBridges('code()\n```', { bridgeClose: '\n```' }), 'code()'); + assert.equal( + stripBridges('```ts\ncode()\n```', { bridgeOpen: '```ts\n', bridgeClose: '\n```' }), + 'code()', + ); +}); + +test('stripBridges tolerates a rewritten fence info string', () => { + // Discord normalized the info line (e.g. a mention) so the exact match fails, + // but the fence marker run still identifies the synthetic opener line. + assert.equal(stripBridges('```@Alice x\ncode()', { bridgeOpen: '```<@1> x\n' }), 'code()'); +}); + +test('stripBridges leaves unrelated text alone', () => { + assert.equal(stripBridges('plain text', { bridgeOpen: '```\n', bridgeClose: '\n```' }), 'plain text'); + assert.equal(stripBridges('plain text', {}), 'plain text'); +}); + +// ── End-to-end shape: split β†’ strip reassembles the original ── + +test('splitting then stripping each part reassembles the original text', () => { + const original = ['# Doc', '```python', ...Array.from({ length: 120 }, (_, i) => `line_${i} = ${i} # padding padding padding`), '```', 'tail **bold text** end'].join('\n'); + assert.ok(original.length > 2000); + const { chunks } = splitPreservingMarkdown(original, 2000); + assert.ok(chunks.length > 1); + for (const c of chunks) assert.ok(c.text.length <= 2000); + const reassembled = chunks + .map((c) => stripBridges(c.text, { bridgeOpen: c.bridgeOpen, bridgeClose: c.bridgeClose })) + .join(''); + assert.equal(reassembled, original); +}); From 0557577dcafeaa96b8cc0b15eb372ab6c88bc292 Mon Sep 17 00:00:00 2001 From: wolframs91 Date: Wed, 8 Jul 2026 22:07:01 +0200 Subject: [PATCH 2/3] fix(relay): close the own-echo/attribution race exposed by live verification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live-tested RFC-006 on a real guild (scripts/rfc6-live.mjs, new): 3 of 21 checks failed, all in the live-event path β€” - Split-send parts were recorded only after the whole batch resolved, but each part's gateway create echo arrives while later parts are still posting: live events went out with bridge markers unstripped and no partOf. sendMany now fires an onSent callback per posted part and the relay records attribution/bridge metadata immediately. - Even per-part, the gateway echo can beat the REST response, so onDiscordMessage now holds an own-webhook echo that has no store ref until the send path records and flushes it (5s timer as safety net). This also fixes the pre-existing single-send attribution race. - editMessage removed surplus-part store refs itself, so the gateway's delete echo emitted raw Discord ids instead of relay ids. Cleanup is now left to the echo, exactly like delete_message. Re-run: 21/21 live checks green; 227 relay unit tests (2 new for onSent). Co-Authored-By: Claude Fable 5 --- ...AL-RFC-006-split-sends-and-inline-audio.md | 17 +- portal-relay/scripts/rfc6-live.mjs | 180 ++++++++++++++++++ portal-relay/src/relay.ts | 100 +++++++--- portal-relay/src/webhook-pool.ts | 15 +- portal-relay/test/split-send.test.ts | 29 +++ 5 files changed, 307 insertions(+), 34 deletions(-) create mode 100644 portal-relay/scripts/rfc6-live.mjs diff --git a/PORTAL-RFC-006-split-sends-and-inline-audio.md b/PORTAL-RFC-006-split-sends-and-inline-audio.md index 42376b6..073c8c3 100644 --- a/PORTAL-RFC-006-split-sends-and-inline-audio.md +++ b/PORTAL-RFC-006-split-sends-and-inline-audio.md @@ -42,6 +42,16 @@ so a code fence or emphasis straddling a boundary breaks everything after it. fetch_history, pins) removes the synthetic markers, so agents always see their original unbroken markdown. A fence reopener whose info string Discord rewrote (mentions in `cleanContent`) is matched on the fence marker run alone. +- **Echo/record race** (found in live verification): the gateway's create echo + for our own webhook message can beat the sending RPC's `store.record` β€” for + split sends it's near-certain (part 1's echo lands while later parts are + still posting), which would dispatch live events unstripped and without + `partOf`. Two-sided fix: `sendMany` fires an `onSent` callback per part so + the relay records each part the moment it exists, and `onDiscordMessage` + HOLDS an own-webhook echo with no store ref until the send path records and + flushes it (`flushEcho`; 5 s timer as safety net). Symmetrically, surplus + parts deleted by an edit leave store cleanup to the gateway's delete echo + (like `delete_message`), so the `message_delete` event keeps its relay id. - **Edit** (`edit_message`): redirected to the first part; new content is re-split and written across the existing parts; surplus parts are deleted; bridge metadata is rewritten (`setSplitMeta`). Needing MORE parts than the @@ -54,7 +64,12 @@ so a code fence or emphasis straddling a boundary breaks everything after it. **Verified:** `portal-relay/test/discord-markdown.test.ts` (140 tests, ported from chapterx) and `test/split-send.test.ts` (sendMany ordering/contiguity/ -partial failure, store round-trip, stripBridges, splitβ†’strip reassembly). +partial failure, onSent per-part callback, store round-trip, stripBridges, +splitβ†’strip reassembly). **Live-verified** (2026-07-08) against a real guild +with `scripts/rfc6-live.mjs` β€” 21/21 checks: 4-part split send, stripped live +events + history + reassembly, `partOf` back-links, shrink edit, collapse with +surplus deletion, grow-edit refusal, delete fan-out, audio attachment CDN +round-trip, native 🐘 reaction. ## 2. Inline audio for agents (mcpl) + protocol audio metadata diff --git a/portal-relay/scripts/rfc6-live.mjs b/portal-relay/scripts/rfc6-live.mjs new file mode 100644 index 0000000..f730c5a --- /dev/null +++ b/portal-relay/scripts/rfc6-live.mjs @@ -0,0 +1,180 @@ +// Live RFC-006 verification: markdown-preserving split sends + audio attachments. +// Starts the relay (one bot) against a real guild, connects the two test personas, +// and exercises the whole split lifecycle plus an audio round-trip. +// +// Side effects (in the target channel): posts a handful of test messages, edits +// and deletes them, uploads one small .wav, adds one 🐘 reaction. Everything it +// creates it deletes at the end. +// +// Usage: DISCORD_TOKEN=... node scripts/rfc6-live.mjs +import { Relay } from '../dist/src/relay.js'; +import { PortalClient } from '../../portal-client/dist/src/index.js'; +import { normalizeAudioMime, audioMimeFor } from '../../portal-mcpl/dist/src/server.js'; + +const [guildId, channelId] = process.argv.slice(2); +const token = process.env.DISCORD_TOKEN; +if (!token || !guildId || !channelId) { + console.error('usage: DISCORD_TOKEN=... node scripts/rfc6-live.mjs '); + process.exit(2); +} + +const config = { + discordToken: token, + wsPort: 8792, + avatarBaseUrl: '', + guildIds: [guildId], + identityPath: new URL('../identity.test.json', import.meta.url).pathname, + permissionsPath: new URL('../permissions.test.json', import.meta.url).pathname, + rolePool: { size: 2, prefix: 'portal-' }, + webhookPoolSize: 1, + heartbeatIntervalMs: 30000, + guildMembersIntent: false, + watchConfig: false, +}; + +const log = (...a) => console.log('[rfc6]', ...a); +let pass = 0; +let fail = 0; +const check = (cond, label) => { + if (cond) { pass++; log(' βœ…', label); } + else { fail++; log(' ❌', label); } +}; +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +/** Wait until fn() returns truthy (polling), or time out. */ +async function until(fn, ms = 10000, step = 250) { + const deadline = Date.now() + ms; + for (;;) { + const v = fn(); + if (v) return v; + if (Date.now() > deadline) return undefined; + await sleep(step); + } +} + +/** Minimal valid WAV: 0.5 s of 440 Hz sine, 8 kHz mono 16-bit (~8 KB). */ +function makeWav() { + const rate = 8000; + const n = rate / 2; + const data = Buffer.alloc(n * 2); + for (let i = 0; i < n; i++) data.writeInt16LE(Math.round(Math.sin((2 * Math.PI * 440 * i) / rate) * 12000), i * 2); + const h = Buffer.alloc(44); + h.write('RIFF', 0); h.writeUInt32LE(36 + data.length, 4); h.write('WAVE', 8); + h.write('fmt ', 12); h.writeUInt32LE(16, 16); h.writeUInt16LE(1, 20); h.writeUInt16LE(1, 22); + h.writeUInt32LE(rate, 24); h.writeUInt32LE(rate * 2, 28); h.writeUInt16LE(2, 32); h.writeUInt16LE(16, 34); + h.write('data', 36); h.writeUInt32LE(data.length, 40); + return Buffer.concat([h, data]); +} + +/** Over-limit content: prose + a long ts fence + a tail, so at least one split + * boundary falls INSIDE the fence (the bridge open/close machinery must fire). */ +const fenceBody = Array.from({ length: 110 }, (_, i) => `const line_${String(i).padStart(3, '0')} = 'markdown split survival test padding';`).join('\n'); +const ORIGINAL = `**RFC-006 live test** β€” the fence below must survive splitting intact.\n\`\`\`ts\n${fenceBody}\n\`\`\`\nTail after the fence, with *emphasis* intact.`; + +async function main() { + const relay = new Relay(config); + await relay.start(); + log('relay started; waiting for caches…'); + await sleep(4000); + + const mythos = new PortalClient({ url: 'ws://127.0.0.1:8792', token: 'tok-mythos', personaId: 'mythos' }); + const lena = new PortalClient({ url: 'ws://127.0.0.1:8792', token: 'tok-lena', personaId: 'lena' }); + await mythos.connect(); + await lena.connect(); + await lena.subscribe(channelId); + + const events = []; // lena's message_create events in the channel + const deletes = []; // lena's message_delete events + lena.on('message', (e) => { if (e.message.channelId === channelId) events.push(e.message); }); + lena.on('messageDelete', (e) => { if (e.channelId === channelId) deletes.push(e.messageId); }); + + // ── 1. split send ── + log(`1. split send (${ORIGINAL.length} chars)`); + const sent = await mythos.sendMessage({ channelId, content: ORIGINAL }); + check(Array.isArray(sent.messageIds) && sent.messageIds.length >= 2, `split into ${sent.messageIds?.length} parts`); + check(sent.messageId === sent.messageIds?.[0], 'messageId === first part'); + + // ── 2. live events: parts arrive stripped, continuations tagged partOf ── + const parts = await until(() => { + const got = sent.messageIds.map((id) => events.find((m) => m.id === id)); + return got.every(Boolean) ? got : undefined; + }); + check(!!parts, 'all parts arrived as live events'); + if (parts) { + check(parts[0].partOf === undefined, 'first part has no partOf'); + check(parts.slice(1).every((m) => m.partOf === sent.messageId), 'continuations partOf β†’ first part'); + check(parts.every((m) => ORIGINAL.includes(m.content)), 'every event content is a clean substring of the original (bridges stripped)'); + } + + // ── 3. history: same messages, same stripping, reassembly ── + const hist1 = await mythos.fetchHistory({ channelId, limit: 10 }); + const hParts = sent.messageIds.map((id) => hist1.messages.find((m) => m.id === id)); + check(hParts.every(Boolean), 'all parts present in fetch_history'); + if (hParts.every(Boolean)) { + check(hParts.every((m) => ORIGINAL.includes(m.content)), 'history contents are clean substrings'); + const joined = [hParts.map((m) => m.content).join(''), hParts.map((m) => m.content).join('\n')]; + check(joined.includes(ORIGINAL), 'parts reassemble to the original'); + } + + // ── 4. edit: shrink within part count, then collapse to one part ── + log('2. edit lifecycle'); + const SHRUNK = ORIGINAL.slice(0, 2600) + "';\n```\nshrunk tail."; + await mythos.editMessage(sent.messageId, SHRUNK); + const hist2 = await mythos.fetchHistory({ channelId, limit: 10 }); + const after = sent.messageIds.map((id) => hist2.messages.find((m) => m.id === id)).filter(Boolean); + check(after.length >= 2 && after.length <= sent.messageIds.length, `shrunk edit spans ${after.length} parts`); + check(after.every((m) => SHRUNK.includes(m.content)), 'edited parts are clean substrings of new content'); + + await mythos.editMessage(sent.messageId, 'collapsed to a single short message.'); + const gone = await until(() => sent.messageIds.slice(1).every((id) => deletes.includes(id) || !id)); + check(!!gone, 'surplus parts deleted on collapse (delete events seen)'); + const hist3 = await mythos.fetchHistory({ channelId, limit: 10 }); + check(!hist3.messages.some((m) => sent.messageIds.slice(1).includes(m.id)), 'surplus parts absent from history'); + + // ── 5. edit that would need MORE parts must be refused ── + const TOO_LONG = ORIGINAL + '\n' + ORIGINAL; + let refused = false; + try { await mythos.editMessage(sent.messageId, TOO_LONG); } catch (err) { refused = /INVALID_PARAMS|shorten/i.test(String(err?.code ?? '') + err.message); } + check(refused, 'growth beyond original part count refused with INVALID_PARAMS'); + + // ── 6. delete fans out over all parts ── + log('3. delete fan-out'); + const sent2 = await mythos.sendMessage({ channelId, content: ORIGINAL }); + check(sent2.messageIds?.length >= 2, `second split send (${sent2.messageIds?.length} parts)`); + deletes.length = 0; + await mythos.deleteMessage(sent2.messageId); + const allGone = await until(() => sent2.messageIds.every((id) => deletes.includes(id))); + check(!!allGone, 'delete of first part removed every part'); + + // ── 7. audio attachment round-trip ── + log('4. audio attachment'); + const wav = makeWav(); + const sentAudio = await mythos.sendMessage({ + channelId, content: 'RFC-006 audio round-trip.', + files: [{ name: 'rfc6-test.wav', bytes: wav.toString('base64'), contentType: 'audio/wav' }], + }); + const audioMsg = await until(() => events.find((m) => m.id === sentAudio.messageId && m.attachments?.length)); + check(!!audioMsg, 'audio message arrived with attachment'); + if (audioMsg) { + const att = audioMsg.attachments[0]; + const mime = audioMimeFor(att); + check(!!mime && mime.startsWith('audio/'), `detected as audio (${att.contentType ?? 'no contentType'} β†’ ${mime})`); + check(normalizeAudioMime('audio/mpeg; rate=44100') === 'audio/mp3', 'MIME normalization sane'); + const res = await fetch(att.url); + const body = Buffer.from(await res.arrayBuffer()); + check(res.ok && body.length === wav.length, `attachment fetchable from CDN (${body.length} bytes)`); + await lena.react(sentAudio.messageId, '🐘', false, true); + check(true, 'native 🐘 reaction accepted'); + } + + // ── cleanup ── + await mythos.deleteMessage(sentAudio.messageId).catch(() => {}); + await mythos.deleteMessage(sent.messageId).catch(() => {}); + log(`done: ${pass} passed, ${fail} failed`); + mythos.close(); + lena.close(); + await relay.stop(); + process.exit(fail ? 1 : 0); +} + +main().catch((err) => { console.error('[rfc6] fatal:', err); process.exit(1); }); diff --git a/portal-relay/src/relay.ts b/portal-relay/src/relay.ts index dca941a..cb57eed 100644 --- a/portal-relay/src/relay.ts +++ b/portal-relay/src/relay.ts @@ -38,6 +38,11 @@ import { AuditLog } from './admin/audit.js'; /** Discord's hard per-message content limit. */ const DISCORD_MSG_LIMIT = 2000; +/** How long to hold an own-webhook send echo waiting for the RPC to record its + * attribution before dispatching it anyway (safety net; normally flushed in + * milliseconds). Generous enough to ride out a slow multi-part send. */ +const ECHO_HOLD_MS = 5000; + export class Relay implements GatewayHooks { private bot: DiscordBot; readonly identity: IdentityStore; @@ -52,6 +57,9 @@ export class Relay implements GatewayHooks { private history: HistoryCache; private gateway: Gateway; private mirror: MirrorCache; + /** Own-webhook send echoes held until their RPC records attribution (the + * gateway event can beat the REST response). See onDiscordMessage. */ + private pendingEchoes = new Map(); private admin?: AdminServer; /** Shared audit log (RFC-005). Present only when the admin panel is enabled; * self-service ops (claim_invite / rotate_token) audit here too. */ @@ -168,6 +176,8 @@ export class Relay implements GatewayHooks { } async stop(): Promise { + for (const held of this.pendingEchoes.values()) clearTimeout(held.timer); + this.pendingEchoes.clear(); this.identity.stopWatching(); this.permissions.stopWatching(); this.invites?.stopWatching(); @@ -767,6 +777,7 @@ export class Relay implements GatewayHooks { personaId, webhookId, }); + this.flushEcho(messageId); return { messageId: ref.relayId }; } @@ -778,50 +789,51 @@ export class Relay implements GatewayHooks { content: c.text, files: i === chunks.length - 1 ? p.files : undefined, })); + // Record each part the moment it is posted (not after the whole batch): + // a part's gateway echo arrives while later parts are still sending, and + // it must already find the bridge metadata and attribution. + const partIds: string[] = []; + const refs: MessageRef[] = []; + const onSent = (i: number, discordMsgId: string, webhookId: string): void => { + partIds.push(discordMsgId); + refs.push( + this.store.record({ + channelId: target.parentChannelId, + threadId: target.threadId, + guildId, + discordMsgId, + personaId, + webhookId, + bridgeOpen: chunks[i].bridgeOpen, + bridgeClose: chunks[i].bridgeClose, + partOf: i > 0 ? partIds[0] : undefined, + }), + ); + this.flushEcho(discordMsgId); + }; let sent: { messageIds: string[]; webhookId: string }; try { - sent = await this.webhooks.sendMany(target.parentChannelId, personaId, optsList); + sent = await this.webhooks.sendMany(target.parentChannelId, personaId, optsList, onSent); } catch (err) { - // Record whatever DID land so the parts stay attributable/editable, and + // Parts that DID land are already recorded by onSent; stamp the (partial) + // part-set on the first so edit/delete still cover the whole set, and // name them in the error so the caller can clean up or continue. if (err instanceof PartialSendError && err.sentIds.length) { - const partial = this.recordParts(target, guildId, personaId, err.webhookId, err.sentIds, chunks); + this.store.setSplitMeta(err.sentIds[0], splitMetaFor(err.sentIds, 0, chunks)); throw rpcError( 'DISCORD_ERROR', `split send failed after ${err.sentIds.length}/${chunks.length} parts ` + - `(${err.reason.message}); posted parts: ${partial.map((r) => r.relayId).join(', ')} ` + + `(${err.reason.message}); posted parts: ${refs.map((r) => r.relayId).join(', ')} ` + `β€” delete them before retrying, or send the remainder separately`, ); } throw rpcError('DISCORD_ERROR', `split send failed: ${(err as Error).message}`); } - const refs = this.recordParts(target, guildId, personaId, sent.webhookId, sent.messageIds, chunks); + // The full part list is only known now β€” stamp it on the first part. + this.store.setSplitMeta(sent.messageIds[0], splitMetaFor(sent.messageIds, 0, chunks)); return { messageId: refs[0].relayId, messageIds: refs.map((r) => r.relayId) }; } - /** Record every part of a split send: bridge strings for later stripping, - * `parts` on the first part, `partOf` back-links on continuations. */ - private recordParts( - target: { parentChannelId: string; threadId?: string }, - guildId: string | null, - personaId: string, - webhookId: string, - discordMsgIds: string[], - chunks: ChunkPiece[], - ): MessageRef[] { - return discordMsgIds.map((discordMsgId, i) => - this.store.record({ - channelId: target.parentChannelId, - threadId: target.threadId, - guildId, - discordMsgId, - personaId, - webhookId, - ...splitMetaFor(discordMsgIds, i, chunks), - }), - ); - } - /** The first part of a split send (edits/deletes operate on the whole set). * For a standalone message this is the ref itself. */ private primaryOf(ref: MessageRef): MessageRef { @@ -858,7 +870,9 @@ export class Relay implements GatewayHooks { for (const id of partIds.slice(chunks.length)) { try { await this.webhooks.delete(webhookId, id, primary.threadId); - this.store.remove(id); + // Store cleanup is left to the gateway's delete echo (onDiscordDelete), + // exactly like delete_message β€” removing the ref here would strip that + // echo of its relay id and channel attribution. } catch (err) { // Keep the ref β€” the message is still live in Discord; dropping the // store row would orphan it beyond any future edit/delete. @@ -932,11 +946,39 @@ export class Relay implements GatewayHooks { active: this.gateway.activePersonas(), })); } + this.history.invalidate(inc.channelId); // new message changes the latest page + // Our own webhook's create echo can beat the sending RPC's store.record + // (gateway push vs REST response race β€” near-certain for split sends, where + // part 1's echo lands while later parts are still posting). Dispatching now + // would leak unstripped bridge markers and misattribute the author, so hold + // the echo; the send path flushes it the moment attribution is recorded. + // The timer is a safety net for webhook messages that never get recorded. + if (inc.webhookId && this.bot.ownsWebhook(inc.webhookId) && !this.store.getByDiscordId(inc.id)) { + const timer = setTimeout(() => { + this.pendingEchoes.delete(inc.id); + this.dispatchDiscordMessage(inc); + }, ECHO_HOLD_MS); + this.pendingEchoes.set(inc.id, { inc, timer }); + return; + } + this.dispatchDiscordMessage(inc); + } + + private dispatchDiscordMessage(inc: IncomingMessage): void { this.history.invalidate(inc.channelId); // new message changes the latest page const { message, authorPersonaId } = this.buildPortalMessage(inc); this.deliverMessage('message_create', message, authorPersonaId); } + /** Release a held own-send echo now that its attribution is recorded. */ + private flushEcho(discordMsgId: string): void { + const held = this.pendingEchoes.get(discordMsgId); + if (!held) return; + clearTimeout(held.timer); + this.pendingEchoes.delete(discordMsgId); + this.dispatchDiscordMessage(held.inc); + } + /** Inbound (human/bot) edit β†’ message_update to interested personas. */ private onDiscordEdit(inc: IncomingMessage): void { this.history.invalidate(inc.channelId); diff --git a/portal-relay/src/webhook-pool.ts b/portal-relay/src/webhook-pool.ts index eb2938e..83996af 100644 --- a/portal-relay/src/webhook-pool.ts +++ b/portal-relay/src/webhook-pool.ts @@ -117,23 +117,30 @@ export class WebhookPool { /** Send several messages as ONE queue item on the persona's webhook, so the * parts of a split send stay contiguous (no same-webhook send can land - * between them). On a mid-sequence failure, rejects with a `PartialSendError` - * carrying the ids that DID go out so the caller can still record them. */ + * between them). `onSent` fires per part the moment it is posted β€” before the + * batch resolves β€” so the caller can record attribution while later parts are + * still in flight (the part's gateway echo races the batch). On a + * mid-sequence failure, rejects with a `PartialSendError` carrying the ids + * that DID go out so the caller can still record them. */ async sendMany( parentChannelId: string, personaId: string, optsList: WebhookSendOpts[], + onSent?: (index: number, messageId: string, webhookId: string) => void, ): Promise<{ messageIds: string[]; webhookId: string }> { const pool = await this.ensurePool(parentChannelId); const webhookId = this.pick(pool, personaId); const task = async (): Promise => { const ids: string[] = []; - for (const opts of optsList) { + for (let i = 0; i < optsList.length; i++) { + let messageId: string; try { - ids.push((await this.ops.sendWebhook(webhookId, opts)).messageId); + messageId = (await this.ops.sendWebhook(webhookId, optsList[i])).messageId; } catch (err) { throw new PartialSendError(ids, webhookId, err as Error); } + ids.push(messageId); + onSent?.(i, messageId, webhookId); } return ids; }; diff --git a/portal-relay/test/split-send.test.ts b/portal-relay/test/split-send.test.ts index e426bf9..0ec1f65 100644 --- a/portal-relay/test/split-send.test.ts +++ b/portal-relay/test/split-send.test.ts @@ -59,6 +59,35 @@ test('sendMany keeps parts contiguous against concurrent sends', async () => { assert.equal(order[i1 + 1], 'a2'); }); +test('sendMany fires onSent per part, in order, as each part posts', async () => { + const { ops } = fakeOps(); + const pool = new WebhookPool(ops, 1); + const base = { username: 'u', avatarURL: '' }; + const seen: Array<[number, string, string]> = []; + await pool.sendMany( + 'chan', 'p1', + [{ ...base, content: 'one' }, { ...base, content: 'two' }], + (i, id, wh) => seen.push([i, id, wh]), + ); + assert.deepEqual(seen, [[0, 'm1', 'wh1'], [1, 'm2', 'wh1']]); +}); + +test('sendMany fires onSent only for parts that landed before a failure', async () => { + const { ops } = fakeOps(1); // second send fails + const pool = new WebhookPool(ops, 1); + const base = { username: 'u', avatarURL: '' }; + const seen: string[] = []; + await assert.rejects( + pool.sendMany( + 'chan', 'p1', + [{ ...base, content: 'one' }, { ...base, content: 'two' }], + (_i, id) => seen.push(id), + ), + PartialSendError, + ); + assert.deepEqual(seen, ['m1']); +}); + test('sendMany surfaces a PartialSendError carrying the ids that DID send', async () => { const { ops } = fakeOps(1); // second send fails const pool = new WebhookPool(ops, 1); From 0f5f5d9c7dfa15478aea1c7ea95fe402cc326210 Mon Sep 17 00:00:00 2001 From: wolframs91 Date: Wed, 8 Jul 2026 23:00:21 +0200 Subject: [PATCH 3/3] test(live): audio carrying verified end-to-end with MiMo v2.5 behind a persona scripts/mimo-audio-live.mjs: one persona posts a spoken .wav, a second persona driven by xiaomi/mimo-v2.5 (OpenRouter, provider order pinned like chapterx's mimo.yaml) consumes it through portal-mcpl's real buildContent (now exported for live verification) mapped to an input_audio part exactly as membrane's adapter does. The model quoted the spoken sentence verbatim. Co-Authored-By: Claude Fable 5 --- ...AL-RFC-006-split-sends-and-inline-audio.md | 7 +- portal-mcpl/src/server.ts | 5 +- portal-relay/scripts/mimo-audio-live.mjs | 156 ++++++++++++++++++ 3 files changed, 165 insertions(+), 3 deletions(-) create mode 100644 portal-relay/scripts/mimo-audio-live.mjs diff --git a/PORTAL-RFC-006-split-sends-and-inline-audio.md b/PORTAL-RFC-006-split-sends-and-inline-audio.md index 073c8c3..c4356b9 100644 --- a/PORTAL-RFC-006-split-sends-and-inline-audio.md +++ b/PORTAL-RFC-006-split-sends-and-inline-audio.md @@ -106,4 +106,9 @@ as URL text notes β€” an audio-capable model behind portal could never hear them lines there). **Verified:** `portal-mcpl/test/audio.test.ts` (opt-in toggle + serialization, -MIME normalization, detection fallback chain). +MIME normalization, detection fallback chain). **Live-verified end-to-end with +a real model** (2026-07-08, `scripts/mimo-audio-live.mjs`): a spoken .wav +posted by one persona reached a MiMo v2.5-driven persona as an mcpl audio +block (`buildContent`, exported for this), was mapped to an OpenRouter +`input_audio` part (membrane's adapter shape), and the model quoted the spoken +sentence verbatim in its reply. diff --git a/portal-mcpl/src/server.ts b/portal-mcpl/src/server.ts index 037eaff..4637567 100644 --- a/portal-mcpl/src/server.ts +++ b/portal-mcpl/src/server.ts @@ -553,8 +553,9 @@ function attachmentNote(att: PortalMessage['attachments'][number]): string { /** Build MCPL content blocks for a message: the text line, plus inlined image * attachments (so the agent can actually see them), inlined audio when the * channel is opted in (so it can hear them), and notes for the rest. - * Best-effort β€” a failed fetch degrades to a text note, never drops the msg. */ -async function buildContent( + * Best-effort β€” a failed fetch degrades to a text note, never drops the msg. + * Exported for live verification (scripts drive a real model with its output). */ +export async function buildContent( m: PortalMessage, opts: { inlineAudio?: boolean } = {}, ): Promise { diff --git a/portal-relay/scripts/mimo-audio-live.mjs b/portal-relay/scripts/mimo-audio-live.mjs new file mode 100644 index 0000000..5c4e1bc --- /dev/null +++ b/portal-relay/scripts/mimo-audio-live.mjs @@ -0,0 +1,156 @@ +// Live audio-carrying test with a REAL model behind a persona. +// +// Mythos posts a spoken .wav into the channel addressing Lena; Lena is driven +// by MiMo v2.5 (OpenRouter): her inbound message is turned into MCPL content +// blocks by portal-mcpl's actual buildContent (audio inlined), mapped to +// OpenRouter `input_audio` parts exactly like membrane's adapter, and MiMo's +// reply is posted back through the persona. Passing = the reply quotes the +// spoken phrase, proving the clip survived Discord β†’ relay β†’ mcpl β†’ model. +// +// The exchange is left in the channel on purpose so humans can see it. +// +// Usage: DISCORD_TOKEN=... OPENROUTER_API_KEY=... \ +// node scripts/mimo-audio-live.mjs +import { readFileSync } from 'node:fs'; +import { basename } from 'node:path'; +import { Relay } from '../dist/src/relay.js'; +import { PortalClient } from '../../portal-client/dist/src/index.js'; +import { buildContent, normalizeAudioMime } from '../../portal-mcpl/dist/src/server.js'; + +const [guildId, channelId, clipPath] = process.argv.slice(2); +const token = process.env.DISCORD_TOKEN; +const orKey = process.env.OPENROUTER_API_KEY; +if (!token || !orKey || !guildId || !channelId || !clipPath) { + console.error('usage: DISCORD_TOKEN=... OPENROUTER_API_KEY=... node scripts/mimo-audio-live.mjs '); + process.exit(2); +} + +const MODEL = 'xiaomi/mimo-v2.5'; +// Pin inference to non-Chinese hosts β€” mirrors config/bots/mimo.yaml. +const PROVIDER = { order: ['parasail', 'venice', 'deepinfra'], allow_fallbacks: false }; +const PHRASE_WORDS = ['elephant', 'midnight', 'copper']; + +const config = { + discordToken: token, + wsPort: 8793, + avatarBaseUrl: '', + guildIds: [guildId], + identityPath: new URL('../identity.test.json', import.meta.url).pathname, + permissionsPath: new URL('../permissions.test.json', import.meta.url).pathname, + rolePool: { size: 2, prefix: 'portal-' }, + webhookPoolSize: 1, + heartbeatIntervalMs: 30000, + guildMembersIntent: false, + watchConfig: false, +}; + +const log = (...a) => console.log('[mimo-live]', ...a); + +/** MCPL audio block MIME β†’ OpenRouter input_audio format (membrane's mapping). */ +function formatFor(mimeType) { + const m = normalizeAudioMime(mimeType); + if (m === 'audio/mp3') return 'mp3'; + if (['audio/wav', 'audio/x-wav', 'audio/wave', 'audio/vnd.wave'].includes(m)) return 'wav'; + return undefined; +} + +/** MCPL content blocks β†’ OpenRouter chat-completions content parts. */ +function toOpenRouterParts(blocks) { + const parts = []; + for (const b of blocks) { + if (b.type === 'text') parts.push({ type: 'text', text: b.text }); + else if (b.type === 'audio') { + const format = formatFor(b.mimeType ?? 'audio/mpeg'); + if (format) parts.push({ type: 'input_audio', input_audio: { data: b.data, format } }); + else log(`skipping audio block with unmappable MIME ${b.mimeType}`); + } + } + return parts; +} + +async function askMimo(parts) { + const res = await fetch('https://openrouter.ai/api/v1/chat/completions', { + method: 'POST', + headers: { Authorization: `Bearer ${orKey}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model: MODEL, + provider: PROVIDER, + max_tokens: 512, + messages: [ + { + role: 'system', + content: + 'You are Lena, a Discord persona. A user sent you an audio clip. ' + + 'Describe exactly what you hear and quote any spoken words verbatim. Reply briefly.', + }, + { role: 'user', content: parts }, + ], + }), + }); + const body = await res.json(); + if (!res.ok) throw new Error(`OpenRouter ${res.status}: ${JSON.stringify(body).slice(0, 300)}`); + return body.choices?.[0]?.message?.content ?? ''; +} + +async function main() { + const relay = new Relay(config); + await relay.start(); + log('relay started; waiting for caches…'); + await new Promise((r) => setTimeout(r, 4000)); + + const mythos = new PortalClient({ url: 'ws://127.0.0.1:8793', token: 'tok-mythos', personaId: 'mythos' }); + const lena = new PortalClient({ url: 'ws://127.0.0.1:8793', token: 'tok-lena', personaId: 'lena' }); + await mythos.connect(); + await lena.connect(); + await lena.subscribe(channelId); + + // Lena's brain: audio message in β†’ mcpl content blocks β†’ MiMo β†’ persona reply. + let replied; + const gotReply = new Promise((resolve) => (replied = resolve)); + lena.on('message', (e) => { + const m = e.message; + if (m.channelId !== channelId || !m.attachments?.length) return; + void (async () => { + const blocks = await buildContent(m, { inlineAudio: true }); + const kinds = blocks.map((b) => b.type).join(', '); + log(`lena received "${m.author.displayName ?? m.author.kind}: ${m.content}" β†’ mcpl blocks: [${kinds}]`); + if (!blocks.some((b) => b.type === 'audio')) { + log('no audio block was inlined β€” aborting'); + return replied({ ok: false, reply: '' }); + } + const reply = await askMimo(toOpenRouterParts(blocks)); + log('mimo replied:', JSON.stringify(reply)); + await lena.sendMessage({ channelId, content: reply || '(empty reply)', replyToId: m.id }); + replied({ ok: true, reply }); + })().catch((err) => { + log('agent error:', err.message); + replied({ ok: false, reply: String(err.message) }); + }); + }); + + const wav = readFileSync(clipPath); + log(`mythos posts the clip (${wav.length} bytes)…`); + await mythos.sendMessage({ + channelId, + content: 'Lena β€” what do you hear in this clip?', + mentionPersonaIds: ['lena'], + files: [{ name: basename(clipPath), bytes: wav.toString('base64'), contentType: 'audio/wav' }], + }); + + const result = await Promise.race([ + gotReply, + new Promise((r) => setTimeout(() => r({ ok: false, reply: '(timeout)' }), 90000)), + ]); + + const heard = PHRASE_WORDS.filter((w) => result.reply.toLowerCase().includes(w)); + log(result.ok && heard.length + ? `βœ… PASS β€” MiMo heard the clip (matched: ${heard.join(', ')})` + : `❌ FAIL β€” ok=${result.ok}, phrase words matched: ${heard.join(', ') || 'none'}`); + + mythos.close(); + lena.close(); + await relay.stop(); + process.exit(result.ok && heard.length ? 0 : 1); +} + +main().catch((err) => { console.error('[mimo-live] fatal:', err); process.exit(1); });