feat: audio attachment support for audio-capable models (Gemini)

config/bots/claude.yaml.example

@@ -28,6 +28,23 @@ authorized_roles: []  # Roles authorized for .history commands (empty = all allo
 include_images: true
 max_images: 5
 
+# Audio configuration (OFF by default; enable only for audio-capable models)
+# Routes Discord audio attachments (mp3/wav/ogg/flac/...) to the model as inline
+# audio. Most models cannot process audio — enable this only for ones that can
+# (e.g. Gemini 3.5 Flash). Audio is sent inline (base64), so it is size-capped.
+# include_audio: true   # default: false
+# max_audio: 1          # max audio files per context (default: 1; 0 disables)
+# oversized_audio_emote: "🐘"  # reaction on messages whose audio exceeds the size cap ('' disables)
+#
+# Sizing: each audio file is capped at 12 MB raw (MAX_AUDIO_BYTES); base64 inflates
+# that to ~16 MB on the wire, just under the ~20 MB request ceiling of inline-audio
+# APIs (Gemini inlineData, OpenRouter input_audio). The cap is PER FILE and the
+# whole context ships in ONE request, so max_audio multiplies the payload:
+# keep max_audio: 1 while the per-file cap is in the 12-15 MB range. If you raise
+# max_audio, budget ~15 MB of raw audio TOTAL and divide by max_audio (e.g.
+# max_audio: 3 -> only feasible if files stay <= ~5 MB each; larger clips will
+# push the request over the provider ceiling and the LLM call will 400).
+
 # Tool configuration
 tools_enabled: true
 tool_output_visible: false  # Show tool calls in Discord

src/agent/loop.ts

@@ -1483,6 +1483,9 @@ export class AgentLoop {
       // Cap image fetching to prevent RAM bloat in image-heavy channels
       // Use 2x max_images to give context builder selection room while preventing worst-case loading
       const maxImagesFetch = Math.max((preConfig.max_images || 5) * 2, 10)
+      // Only fetch audio for audio-capable bots (include_audio); others pass 0.
+      // Use ?? so an explicit max_audio:0 disables fetching (matches the send-side cap).
+      const maxAudioFetch = preConfig.include_audio ? (preConfig.max_audio ?? 1) : 0
       const discordContext = await this.connector.fetchContext({
         channelId,
         depth: fetchDepth,
@@ -1493,6 +1496,8 @@ export class AgentLoop {
         authorized_roles: [],  // Will apply after loading config
         pinnedConfigs,  // Reuse pre-fetched pinned configs (avoids second API call)
         maxImages: maxImagesFetch,  // Prevents loading all images from image-heavy channels
+        maxAudio: maxAudioFetch,  // Only audio-capable bots fetch audio
+        oversizedAudioEmote: preConfig.oversized_audio_emote,
       })
       endProfile('fetchContext')
 

src/config/system.ts

@@ -296,6 +296,10 @@ export class ConfigSystem {
       generate_images: config.generate_images,
       provider_params: config.provider_params,
 
+      // Audio config (off unless explicitly enabled for audio-capable models)
+      include_audio: config.include_audio ?? false,
+      max_audio: config.max_audio ?? 1,
+
       // Text attachment config
       include_text_attachments: config.include_text_attachments ?? true,
       max_text_attachment_kb: config.max_text_attachment_kb || 100,  // 100KB default
@@ -338,6 +342,7 @@ export class ConfigSystem {
       // Loop prevention
       max_bot_reply_chain_depth: config.max_bot_reply_chain_depth ?? 2,
       bot_reply_chain_depth_emote: config.bot_reply_chain_depth_emote || '🔁',
+      oversized_audio_emote: config.oversized_audio_emote ?? '🐘',
 
       // Message filtering
       ignore_dotted_messages: config.ignore_dotted_messages !== false,  // Default: true

src/context/builder.ts

@@ -125,7 +125,8 @@ export class ContextBuilder {
       discordContext.documents,
       config,
       botDiscordUsername,
-      imageSelectionMarker
+      imageSelectionMarker,
+      discordContext.audios
     )
 
     logger.debug({

src/context/stages/format-messages.ts

@@ -11,6 +11,7 @@ import type {
   ContentBlock,
   CachedImage,
   CachedDocument,
+  CachedAudio,
   BotConfig,
 } from '../../types.js'
 import { prepareImage, resampleImage, MAX_IMAGE_BASE64_BYTES } from '../image-processing.js'
@@ -29,13 +30,19 @@ export async function formatMessages(
   documents: CachedDocument[],
   config: BotConfig,
   botDiscordUsername?: string,
-  cacheMarkerMessageId?: string | null
+  cacheMarkerMessageId?: string | null,
+  audios: CachedAudio[] = []
 ): Promise<ParticipantMessage[]> {
   const participantMessages: ParticipantMessage[] = []
 
   // Create image lookup
   const imageMap = new Map(images.map((img) => [img.url, img]))
 
+  // Create audio lookup; gate emission by max_audio (oldest dropped first).
+  const audioMap = new Map(audios.map((a) => [a.url, a]))
+  let audioEmitted = 0
+  const maxAudio = config.max_audio ?? 1
+
   // Create document lookup by messageId
   const documentsByMessageId = new Map<string, CachedDocument[]>()
   for (const doc of documents) {
@@ -47,6 +54,11 @@ export async function formatMessages(
 
   const max_images = config.max_images || 5
   const maxTotalBase64Bytes = 15 * 1024 * 1024  // 15 MB total base64 data for images
+  // Combined images+audio inline budget, kept under Gemini's ~20 MB inline-data
+  // request ceiling. Function-scoped so audio can bound itself against the
+  // already-selected image total. `totalBase64Size` accumulates both.
+  const maxInlineBase64Bytes = 18 * 1024 * 1024
+  let totalBase64Size = 0
 
   // Find cache marker position for image selection anchoring
   const cacheMarkerIndex = cacheMarkerMessageId
@@ -73,7 +85,6 @@ export async function formatMessages(
   if (config.include_images) {
     let prefixImageCount = 0
     let ephemeralImageCount = 0
-    let totalBase64Size = 0
 
     // TIER 1: Images in cached prefix (only if cache_images is enabled)
     if (cacheImages) {
@@ -287,6 +298,49 @@ export async function formatMessages(
       }
     }
 
+    // Add audio content for audio-capable models (sent inline as base64)
+    if (config.include_audio && audioEmitted < maxAudio) {
+      for (const attachment of msg.attachments) {
+        if (audioEmitted >= maxAudio) break
+        // Presence in audioMap is the authoritative signal — the connector
+        // already detected audio by content_type OR filename extension (Discord's
+        // content_type is optional, so we must not re-gate on it here).
+        const cached = audioMap.get(attachment.url)
+        if (!cached) continue
+
+        // Bound the combined images+audio inline payload (Gemini ~20 MB ceiling).
+        const audioBase64Bytes = Math.ceil(cached.data.length * 4 / 3)
+        if (totalBase64Size + audioBase64Bytes > maxInlineBase64Bytes) {
+          logger.warn({
+            messageId: msg.id,
+            url: attachment.url,
+            audioMB: (audioBase64Bytes / 1024 / 1024).toFixed(2),
+            usedMB: (totalBase64Size / 1024 / 1024).toFixed(2),
+            capMB: (maxInlineBase64Bytes / 1024 / 1024).toFixed(2),
+          }, 'Skipping audio: would exceed combined inline media budget')
+          continue
+        }
+
+        content.push({
+          type: 'audio',
+          source: {
+            type: 'base64',
+            data: cached.data.toString('base64'),
+            media_type: cached.mediaType,
+          },
+          ...(cached.duration !== undefined ? { duration: cached.duration } : {}),
+        } as ContentBlock)
+        totalBase64Size += audioBase64Bytes
+        audioEmitted++
+        logger.debug({
+          messageId: msg.id,
+          url: attachment.url,
+          mediaType: cached.mediaType,
+          sizeMB: (cached.data.length / 1024 / 1024).toFixed(2),
+        }, 'Added audio to content')
+      }
+    }
+
     // Add text document content in XML blocks
     if (config.include_text_attachments !== false) {
       const maxSizeBytes = (config.max_text_attachment_kb || 200) * 1024