diff --git a/config/bots/claude.yaml.example b/config/bots/claude.yaml.example index f2db3e3..87b9865 100644 --- a/config/bots/claude.yaml.example +++ b/config/bots/claude.yaml.example @@ -28,6 +28,23 @@ authorized_roles: [] # Roles authorized for .history commands (empty = all allo include_images: true max_images: 5 +# Audio configuration (OFF by default; enable only for audio-capable models) +# Routes Discord audio attachments (mp3/wav/ogg/flac/...) to the model as inline +# audio. Most models cannot process audio — enable this only for ones that can +# (e.g. Gemini 3.5 Flash). Audio is sent inline (base64), so it is size-capped. +# include_audio: true # default: false +# max_audio: 1 # max audio files per context (default: 1; 0 disables) +# oversized_audio_emote: "🐘" # reaction on messages whose audio exceeds the size cap ('' disables) +# +# Sizing: each audio file is capped at 12 MB raw (MAX_AUDIO_BYTES); base64 inflates +# that to ~16 MB on the wire, just under the ~20 MB request ceiling of inline-audio +# APIs (Gemini inlineData, OpenRouter input_audio). The cap is PER FILE and the +# whole context ships in ONE request, so max_audio multiplies the payload: +# keep max_audio: 1 while the per-file cap is in the 12-15 MB range. If you raise +# max_audio, budget ~15 MB of raw audio TOTAL and divide by max_audio (e.g. +# max_audio: 3 -> only feasible if files stay <= ~5 MB each; larger clips will +# push the request over the provider ceiling and the LLM call will 400). + # Tool configuration tools_enabled: true tool_output_visible: false # Show tool calls in Discord diff --git a/src/agent/loop.ts b/src/agent/loop.ts index c28cfea..31ef592 100644 --- a/src/agent/loop.ts +++ b/src/agent/loop.ts @@ -1483,6 +1483,9 @@ export class AgentLoop { // Cap image fetching to prevent RAM bloat in image-heavy channels // Use 2x max_images to give context builder selection room while preventing worst-case loading const maxImagesFetch = Math.max((preConfig.max_images || 5) * 2, 10) + // Only fetch audio for audio-capable bots (include_audio); others pass 0. + // Use ?? so an explicit max_audio:0 disables fetching (matches the send-side cap). + const maxAudioFetch = preConfig.include_audio ? (preConfig.max_audio ?? 1) : 0 const discordContext = await this.connector.fetchContext({ channelId, depth: fetchDepth, @@ -1493,6 +1496,8 @@ export class AgentLoop { authorized_roles: [], // Will apply after loading config pinnedConfigs, // Reuse pre-fetched pinned configs (avoids second API call) maxImages: maxImagesFetch, // Prevents loading all images from image-heavy channels + maxAudio: maxAudioFetch, // Only audio-capable bots fetch audio + oversizedAudioEmote: preConfig.oversized_audio_emote, }) endProfile('fetchContext') diff --git a/src/config/system.ts b/src/config/system.ts index 47c4a55..b1b60f0 100644 --- a/src/config/system.ts +++ b/src/config/system.ts @@ -296,6 +296,10 @@ export class ConfigSystem { generate_images: config.generate_images, provider_params: config.provider_params, + // Audio config (off unless explicitly enabled for audio-capable models) + include_audio: config.include_audio ?? false, + max_audio: config.max_audio ?? 1, + // Text attachment config include_text_attachments: config.include_text_attachments ?? true, max_text_attachment_kb: config.max_text_attachment_kb || 100, // 100KB default @@ -338,6 +342,7 @@ export class ConfigSystem { // Loop prevention max_bot_reply_chain_depth: config.max_bot_reply_chain_depth ?? 2, bot_reply_chain_depth_emote: config.bot_reply_chain_depth_emote || '🔁', + oversized_audio_emote: config.oversized_audio_emote ?? '🐘', // Message filtering ignore_dotted_messages: config.ignore_dotted_messages !== false, // Default: true diff --git a/src/context/builder.ts b/src/context/builder.ts index 7d2633a..7e521f1 100644 --- a/src/context/builder.ts +++ b/src/context/builder.ts @@ -125,7 +125,8 @@ export class ContextBuilder { discordContext.documents, config, botDiscordUsername, - imageSelectionMarker + imageSelectionMarker, + discordContext.audios ) logger.debug({ diff --git a/src/context/stages/format-messages.ts b/src/context/stages/format-messages.ts index 68f448c..743d485 100644 --- a/src/context/stages/format-messages.ts +++ b/src/context/stages/format-messages.ts @@ -11,6 +11,7 @@ import type { ContentBlock, CachedImage, CachedDocument, + CachedAudio, BotConfig, } from '../../types.js' import { prepareImage, resampleImage, MAX_IMAGE_BASE64_BYTES } from '../image-processing.js' @@ -29,13 +30,19 @@ export async function formatMessages( documents: CachedDocument[], config: BotConfig, botDiscordUsername?: string, - cacheMarkerMessageId?: string | null + cacheMarkerMessageId?: string | null, + audios: CachedAudio[] = [] ): Promise { const participantMessages: ParticipantMessage[] = [] // Create image lookup const imageMap = new Map(images.map((img) => [img.url, img])) + // Create audio lookup; gate emission by max_audio (oldest dropped first). + const audioMap = new Map(audios.map((a) => [a.url, a])) + let audioEmitted = 0 + const maxAudio = config.max_audio ?? 1 + // Create document lookup by messageId const documentsByMessageId = new Map() for (const doc of documents) { @@ -47,6 +54,11 @@ export async function formatMessages( const max_images = config.max_images || 5 const maxTotalBase64Bytes = 15 * 1024 * 1024 // 15 MB total base64 data for images + // Combined images+audio inline budget, kept under Gemini's ~20 MB inline-data + // request ceiling. Function-scoped so audio can bound itself against the + // already-selected image total. `totalBase64Size` accumulates both. + const maxInlineBase64Bytes = 18 * 1024 * 1024 + let totalBase64Size = 0 // Find cache marker position for image selection anchoring const cacheMarkerIndex = cacheMarkerMessageId @@ -73,7 +85,6 @@ export async function formatMessages( if (config.include_images) { let prefixImageCount = 0 let ephemeralImageCount = 0 - let totalBase64Size = 0 // TIER 1: Images in cached prefix (only if cache_images is enabled) if (cacheImages) { @@ -287,6 +298,49 @@ export async function formatMessages( } } + // Add audio content for audio-capable models (sent inline as base64) + if (config.include_audio && audioEmitted < maxAudio) { + for (const attachment of msg.attachments) { + if (audioEmitted >= maxAudio) break + // Presence in audioMap is the authoritative signal — the connector + // already detected audio by content_type OR filename extension (Discord's + // content_type is optional, so we must not re-gate on it here). + const cached = audioMap.get(attachment.url) + if (!cached) continue + + // Bound the combined images+audio inline payload (Gemini ~20 MB ceiling). + const audioBase64Bytes = Math.ceil(cached.data.length * 4 / 3) + if (totalBase64Size + audioBase64Bytes > maxInlineBase64Bytes) { + logger.warn({ + messageId: msg.id, + url: attachment.url, + audioMB: (audioBase64Bytes / 1024 / 1024).toFixed(2), + usedMB: (totalBase64Size / 1024 / 1024).toFixed(2), + capMB: (maxInlineBase64Bytes / 1024 / 1024).toFixed(2), + }, 'Skipping audio: would exceed combined inline media budget') + continue + } + + content.push({ + type: 'audio', + source: { + type: 'base64', + data: cached.data.toString('base64'), + media_type: cached.mediaType, + }, + ...(cached.duration !== undefined ? { duration: cached.duration } : {}), + } as ContentBlock) + totalBase64Size += audioBase64Bytes + audioEmitted++ + logger.debug({ + messageId: msg.id, + url: attachment.url, + mediaType: cached.mediaType, + sizeMB: (cached.data.length / 1024 / 1024).toFixed(2), + }, 'Added audio to content') + } + } + // Add text document content in XML blocks if (config.include_text_attachments !== false) { const maxSizeBytes = (config.max_text_attachment_kb || 200) * 1024 diff --git a/src/discord/connector.ts b/src/discord/connector.ts index d652964..65e0c26 100644 --- a/src/discord/connector.ts +++ b/src/discord/connector.ts @@ -14,6 +14,7 @@ import { DiscordMessage, CachedImage, CachedDocument, + CachedAudio, DiscordError, } from '../types.js' import { logger } from '../utils/logger.js' @@ -42,6 +43,56 @@ export interface TrackedPin { } const MAX_TEXT_ATTACHMENT_BYTES = 200_000 // ~200 KB of inline text per attachment +// Cap raw audio so the base64-inflated payload stays under Gemini's ~20 MB +// inline-data request ceiling (12 MB raw → ~16 MB base64, leaving headroom). +const MAX_AUDIO_BYTES = 12 * 1024 * 1024 +// Bound the in-memory audio cache (it has no disk backing, unlike images, and +// audio is rarely re-sent so the hit rate is low). LRU-evict oldest entries +// past this budget — enough to dedup audio still in the rolling context window. +const AUDIO_CACHE_MAX_BYTES = 64 * 1024 * 1024 + +// Filename-extension → audio MIME (Gemini-accepted set), used when Discord omits +// content_type — it is OPTIONAL on the attachment object, so we can't rely on it +// alone (mirrors the extension fallback used for text attachments). +const AUDIO_EXTENSION_MIME: Record = { + '.mp3': 'audio/mp3', + '.wav': 'audio/wav', + '.ogg': 'audio/ogg', + '.oga': 'audio/ogg', + '.opus': 'audio/ogg', + '.flac': 'audio/flac', + '.aac': 'audio/aac', + '.aiff': 'audio/aiff', + '.aif': 'audio/aiff', +} + +// Map a Discord-reported audio content type to a MIME string Gemini accepts. +// Discord labels MP3 as "audio/mpeg" (or the legacy aliases "audio/mpeg3" / +// "audio/x-mpeg-3"); Gemini's inline-data MIME for MP3 is "audio/mp3". +function normalizeAudioMime(contentType: string): string { + const base = contentType.split(';')[0]!.trim().toLowerCase() + if (base === 'audio/mpeg' || base === 'audio/mpg' || base === 'audio/mpeg3' || base === 'audio/x-mpeg-3') return 'audio/mp3' + return base +} + +/** True if the attachment is audio — by content_type OR, when Discord omits it, + * by a known audio file extension. */ +function isAudioAttachment(att: Attachment): boolean { + if (att.contentType?.startsWith('audio/')) return true + const name = att.name?.toLowerCase() ?? '' + return Object.keys(AUDIO_EXTENSION_MIME).some((ext) => name.endsWith(ext)) +} + +/** Resolve a Gemini-accepted audio MIME from content_type, falling back to the + * filename extension. Returns null if the type can't be determined. */ +function audioMimeFor(att: Attachment): string | null { + if (att.contentType?.startsWith('audio/')) return normalizeAudioMime(att.contentType) + const name = att.name?.toLowerCase() ?? '' + for (const [ext, mime] of Object.entries(AUDIO_EXTENSION_MIME)) { + if (name.endsWith(ext)) return mime + } + return null +} /** Extract a Unix timestamp (ms) from a Discord snowflake ID */ function snowflakeToTimestamp(id: string): number { @@ -57,6 +108,8 @@ export interface FetchContextParams { authorized_roles?: string[] pinnedConfigs?: string[] // Optional: Pre-fetched pinned configs (skips fetchPinned call) maxImages?: number // Optional: Cap image fetching to avoid RAM bloat (default: unlimited) + maxAudio?: number // Optional: Cap audio fetching (default: 0 — only audio-capable bots fetch audio) + oversizedAudioEmote?: string // Optional: React to messages whose audio was skipped for size (falsy = no reaction) ignoreHistory?: boolean // Optional: Skip .history command processing (raw fetch) } @@ -64,7 +117,11 @@ export class DiscordConnector { private client: Client private typingIntervals = new Map() private imageCache = new Map() + private audioCache = new Map() // URL -> cached audio (in-memory, per session) private urlToFilename = new Map() // URL -> filename for disk cache lookup + // Messages already reacted-to for oversized audio (avoid re-reacting every + // activation — fetchContext rescans the rolling window). Bounded FIFO. + private oversizedAudioReacted = new Set() private urlMapPath: string // Path to URL map file // Push-based caches (populated from gateway events, avoids API fetches) @@ -514,7 +571,7 @@ export class DiscordConnector { * Fetch context from Discord (messages, configs, images) */ async fetchContext(params: FetchContextParams): Promise { - const { channelId, depth, targetMessageId, firstMessageId, authorized_roles, maxImages, ignoreHistory } = params + const { channelId, depth, targetMessageId, firstMessageId, authorized_roles, maxImages, maxAudio, oversizedAudioEmote, ignoreHistory } = params // Profiling helper const timings: Record = {} @@ -773,11 +830,14 @@ export class DiscordConnector { // Download/cache images and fetch text attachments const images: CachedImage[] = [] const documents: CachedDocument[] = [] + const audios: CachedAudio[] = [] let newImagesDownloaded = 0 - logger.debug({ messageCount: messages.length, maxImages }, 'Checking messages for attachments') - + logger.debug({ messageCount: messages.length, maxImages, maxAudio }, 'Checking messages for attachments') + // Track whether we've hit the image cap to avoid unnecessary processing const imageLimitReached = () => maxImages !== undefined && images.length >= maxImages + // Only audio-capable bots pass maxAudio > 0; others never fetch audio. + const audioLimitReached = () => audios.length >= (maxAudio ?? 0) // Iterate newest-first so image cap keeps recent images (context builder wants recent ones) // Messages array is chronological (oldest-first), so we reverse for image fetching @@ -799,6 +859,38 @@ export class DiscordConnector { newImagesDownloaded++ } } + } else if (isAudioAttachment(attachment)) { + if (audioLimitReached()) { + continue + } + // Reject oversized audio up front (Discord reports size) so we never + // buffer a huge upload into memory just to discard it. + if (attachment.size && attachment.size > MAX_AUDIO_BYTES) { + logger.warn({ size: attachment.size, url: attachment.url }, 'Skipping oversized audio attachment (pre-fetch)') + // Signal "file too large" on the message itself (same pattern as + // the 🚫 blocked-message reaction) — once per message, not per scan. + if (oversizedAudioEmote && !this.oversizedAudioReacted.has(msg.id)) { + this.oversizedAudioReacted.add(msg.id) + if (this.oversizedAudioReacted.size > 500) { + const oldest = this.oversizedAudioReacted.values().next().value + if (oldest !== undefined) this.oversizedAudioReacted.delete(oldest) + } + msg.react(oversizedAudioEmote).catch((error) => + logger.warn({ error, messageId: msg.id, emote: oversizedAudioEmote }, 'Failed to react to oversized audio message')) + } + continue + } + // content_type is optional on Discord attachments — resolve the MIME + // from it or the filename extension; skip if undeterminable. + const mediaType = audioMimeFor(attachment) + if (!mediaType) { + logger.debug({ url: attachment.url, name: attachment.name }, 'Audio attachment with undeterminable type, skipping') + continue + } + const cached = await this.cacheAudio(attachment.url, mediaType, msg.id, attachment.duration ?? undefined) + if (cached) { + audios.push(cached) + } } else if (this.isTextAttachment(attachment)) { const doc = await this.fetchTextAttachment(attachment, msg.id) if (doc) { @@ -814,7 +906,7 @@ export class DiscordConnector { } endProfile('attachmentProcessing') - logger.debug({ totalImages: images.length, totalDocuments: documents.length }, 'Attachment processing complete') + logger.debug({ totalImages: images.length, totalDocuments: documents.length, totalAudios: audios.length }, 'Attachment processing complete') // Build inheritance info for plugin state const inheritanceInfo: DiscordContext['inheritanceInfo'] = {} @@ -838,6 +930,7 @@ export class DiscordConnector { messageCount: discordMessages.length, imageCount: images.length, documentCount: documents.length, + audioCount: audios.length, pinnedCount: pinnedConfigs.length, }, '⏱️ PROFILING: fetchContext breakdown (ms)') @@ -846,6 +939,7 @@ export class DiscordConnector { pinnedConfigs, images, documents, + audios, guildId: channel.guildId, inheritanceInfo: Object.keys(inheritanceInfo).length > 0 ? inheritanceInfo : undefined, } @@ -2740,6 +2834,63 @@ export class DiscordConnector { } } + /** + * Download and cache an audio attachment (in-memory, per session). Audio is + * sent inline (base64) to audio-capable models like Gemini, so it is not + * processed/transcoded — just fetched, size-capped, and MIME-normalized. + */ + private async cacheAudio(url: string, mediaType: string, messageId: string, duration?: number): Promise { + const existing = this.audioCache.get(url) + if (existing) { + // Refresh recency (Map keeps insertion order → re-insert moves to newest). + this.audioCache.delete(url) + this.audioCache.set(url, existing) + return existing + } + + try { + const response = await fetch(url) + if (!response.ok) { + logger.warn({ status: response.status, url }, 'Failed to fetch audio attachment') + return null + } + const buffer = Buffer.from(await response.arrayBuffer()) + if (buffer.length > MAX_AUDIO_BYTES) { + logger.warn({ size: buffer.length, url }, 'Skipping oversized audio attachment') + return null + } + + const hash = createHash('sha256').update(buffer).digest('hex') + const cached: CachedAudio = { + url, + messageId, + data: buffer, + mediaType, + hash, + ...(duration !== undefined ? { duration } : {}), + } + this.audioCache.set(url, cached) + this.evictAudioCache() + logger.debug({ url, mediaType, bytes: buffer.length }, 'Cached audio attachment') + return cached + } catch (error) { + logger.warn({ error, url }, 'Failed to download audio attachment') + return null + } + } + + /** LRU-evict oldest audio cache entries once the total exceeds the byte budget. */ + private evictAudioCache(): void { + let total = 0 + for (const a of this.audioCache.values()) total += a.data.length + if (total <= AUDIO_CACHE_MAX_BYTES) return + for (const [key, a] of this.audioCache) { + if (total <= AUDIO_CACHE_MAX_BYTES) break + this.audioCache.delete(key) // oldest first (insertion order) + total -= a.data.length + } + } + private splitMessage(content: string, maxLength: number): string[] { if (content.length <= maxLength) { return [content] diff --git a/src/llm/membrane/adapter.test.ts b/src/llm/membrane/adapter.test.ts index d99ec6e..3ba02d3 100644 --- a/src/llm/membrane/adapter.test.ts +++ b/src/llm/membrane/adapter.test.ts @@ -173,11 +173,41 @@ describe('Content Block Conversion', () => { }, tokenEstimate: 1000, }; - + const result = toMembraneContentBlock(input); expect((result as any).tokenEstimate).toBe(1000); }); }); + + describe('audio blocks', () => { + it('should convert base64 audio with media_type → mediaType', () => { + const input: ContentBlock = { + type: 'audio', + source: { + type: 'base64', + data: 'base64audio', + media_type: 'audio/mp3', + }, + }; + + const result = toMembraneContentBlock(input); + + expect(result.type).toBe('audio'); + expect((result as any).source.type).toBe('base64'); + expect((result as any).source.data).toBe('base64audio'); + expect((result as any).source.mediaType).toBe('audio/mp3'); + expect((result as any).source.media_type).toBeUndefined(); + }); + + it('should preserve duration when present', () => { + const input: any = { + type: 'audio', + source: { type: 'base64', data: 'd', media_type: 'audio/wav' }, + duration: 106, + }; + expect((toMembraneContentBlock(input) as any).duration).toBe(106); + }); + }); describe('tool_use blocks', () => { it('should convert tool_use block', () => { diff --git a/src/llm/membrane/adapter.ts b/src/llm/membrane/adapter.ts index 6dcef8d..7272b94 100644 --- a/src/llm/membrane/adapter.ts +++ b/src/llm/membrane/adapter.ts @@ -164,7 +164,18 @@ export function toMembraneContentBlock(block: ContentBlock): MembraneContentBloc } return imgResult; } - + + case 'audio': + return { + type: 'audio', + source: { + type: 'base64', + data: block.source.data, + mediaType: block.source.media_type, + }, + ...(block.duration !== undefined ? { duration: block.duration } : {}), + } as MembraneContentBlock; + case 'tool_use': return { type: 'tool_use', diff --git a/src/llm/membrane/provider.ts b/src/llm/membrane/provider.ts index 12e45ed..f13ee1e 100644 --- a/src/llm/membrane/provider.ts +++ b/src/llm/membrane/provider.ts @@ -563,7 +563,7 @@ export class MembraneProvider { for (const msg of clone.messages) { if (msg.content && Array.isArray(msg.content)) { for (const block of msg.content) { - if (block.type === 'image' && block.source?.type === 'base64') { + if ((block.type === 'image' || block.type === 'audio') && block.source?.type === 'base64') { block.source.data = `[BASE64_DATA_${block.source.data?.length || 0}_BYTES]`; } } diff --git a/src/types.ts b/src/types.ts index a78b36d..21563a7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -53,6 +53,7 @@ export interface ParticipantMessage { export type ContentBlock = | TextContent | ImageContent + | AudioContent | ToolUseContent | ToolResultContent | ThinkingContent @@ -89,6 +90,16 @@ export interface ImageContent { } } +export interface AudioContent { + type: 'audio' + source: { + type: 'base64' + data: string // base64-encoded audio + media_type: string // 'audio/mp3', 'audio/wav', 'audio/ogg', etc. + } + duration?: number // seconds, if known +} + export interface ToolUseContent { type: 'tool_use' id: string @@ -210,6 +221,10 @@ export interface BotConfig { cache_images?: boolean // If true, include images in cached prefix (requires deterministic handling). Default: false (images only in rolling window) generate_images?: boolean // If true, set responseModalities for image generation models (overrides auto-detect from model name) + // Audio config — only enable for audio-capable models (e.g. Gemini). Default off. + include_audio?: boolean // If true, route audio attachments to this model as inline audio + max_audio?: number // Max audio files to include per context (default: 1 when include_audio) + // Text attachment config include_text_attachments: boolean max_text_attachment_kb: number // Max size per text attachment in KB @@ -251,6 +266,7 @@ export interface BotConfig { // Loop prevention max_bot_reply_chain_depth: number // Max consecutive bot messages in reply chain (prevents bot loops) bot_reply_chain_depth_emote: string // Emote to show when bot reply chain depth limit is reached + oversized_audio_emote: string // Reaction for audio attachments over the size cap ('' disables) // Message filtering ignore_dotted_messages: boolean // If true (default), dot-prefixed messages are hidden from context and don't trigger activation @@ -496,6 +512,7 @@ export interface DiscordContext { pinnedConfigs: string[] // Raw YAML strings from pinned messages images: CachedImage[] documents: CachedDocument[] // Text file contents + audios: CachedAudio[] // Audio file contents (for audio-capable models) guildId: string /** Inheritance info for plugin state */ inheritanceInfo?: { @@ -530,6 +547,15 @@ export interface CachedDocument { truncated?: boolean } +export interface CachedAudio { + url: string + messageId: string + data: Buffer + mediaType: string // normalized, e.g. 'audio/mp3', 'audio/wav', 'audio/ogg' + hash: string + duration?: number // seconds, when Discord provides it (e.g. voice messages) +} + // ============================================================================ // Events // ============================================================================