feat(stt): add energy-based VAD with silence gate and auto-stop (#249)

* loop: stt-warmup completed after 6 iterations

* refactor: extract STT startup helpers and simplify audio recorder

* feat(stt): add energy-based VAD with silence gate and auto-stop

- New VoiceActivityDetector class: adaptive noise floor, per-frame RMS
  classification, trailing-silence endpointing, configurable thresholds
- AudioWorkletProcessor computes RMS in existing flush loop and posts
  {samples, rms} instead of a bare Int16Array
- AudioRecorder integrates VAD via handleVadFrame() in both worklet and
  ScriptProcessor paths; auto-calls stop() on trailing silence; silence
  gate in processRecording() fires onNoSpeech instead of sending empty
  audio to the transcription API (fixes Whisper hallucination bug)
- useSTT wires setOnNoSpeech to reset isProcessing/isRecording cleanly
  without surfacing an error
- computeRms() exported helper for ScriptProcessor fallback path

* fix(stt): clear processing state when stopping a silent recording

stopRecording() called recorder.stop() before setIsProcessing(true). For
a silent take the recorder synchronously fires onNoSpeech (which sets
processing false), so the subsequent setIsProcessing(true) won and the
voice overlay was stuck in the processing state forever.

Set processing before stop() so the recorder callback (onNoSpeech or
onDataAvailable) has the final say on the processing flag.

* fix(stt): always emit a terminal signal so the voice overlay never sticks

Two remaining stuck paths after the silent-stop fix:

- processRecording() early-returned without firing onNoSpeech when a take
  had no captured audio (e.g. a quick start/stop under one worklet flush,
  totalSamples === 0). Now any non-emitting path (empty audio or no
  detected speech) fires onNoSpeech, while a genuine abort still returns
  silently.
- stopRecording() set isProcessing true even when the recorder was no
  longer recording (double-stop or post auto-stop), where stop() skips
  processRecording entirely and nothing clears the flag. Guard the
  external path on recorder state before entering processing.

* loop: mediarecorder-stt completed after 6 iterations

* refactor(stt): extract shared helpers and cleanup

* refactor(stt): add separate arm/disarm thresholds for voice swipe

* fix(stt): prevent default touch behavior on voice gesture handlers

* feat(stt): add audio-level speech detection before emitting data

Author: chriswritescode-dev

frontend/public/audio-worklet-processor.js

@@ -1,5 +1,6 @@
 import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
 import { sttApi } from './stt'
+import { FetchError } from './fetchWrapper'
 
 describe('WAV extension selection logic', () => {
   const originalFetch = global.fetch
@@ -69,6 +70,18 @@ describe('WAV extension selection logic', () => {
     expect(audioFile.name).toBe('recording.m4a')
   })
 
+  it('should return webm for audio/webm;codecs=opus blob', async () => {
+    const blob = new Blob([], { type: 'audio/webm;codecs=opus' })
+    mockFetch.mockResolvedValueOnce(createMockResponse(true, { text: 'test' }))
+
+    await sttApi.transcribe(blob, 'test-user')
+
+    const callArgs = mockFetch.mock.calls[0]
+    const formData = callArgs[1]?.body as FormData
+    const audioFile = formData.get('audio') as File
+    expect(audioFile.name).toBe('recording.webm')
+  })
+
   it('should default to wav for unknown types', async () => {
     const blob = new Blob([], { type: 'audio/unknown' })
     mockFetch.mockResolvedValueOnce(createMockResponse(true, { text: 'test' }))
@@ -123,3 +136,85 @@ describe('WAV extension selection logic', () => {
     expect(formData.get('audio')).toBeInstanceOf(File)
   })
 })
+
+describe('sttApi.transcribe cancellation and timeout', () => {
+  const originalFetch = global.fetch
+
+  beforeEach(() => {
+    vi.stubGlobal('fetch', vi.fn())
+  })
+
+  afterEach(() => {
+    vi.restoreAllMocks()
+    global.fetch = originalFetch
+  })
+
+  it('throws 499 CANCELED when caller aborts', async () => {
+    const blob = new Blob([], { type: 'audio/webm;codecs=opus' })
+    const abortController = new AbortController()
+
+    const mockFetch = global.fetch as ReturnType<typeof vi.fn>
+    mockFetch.mockImplementation((_url: string, options: RequestInit) => {
+      const signal = options.signal as AbortSignal
+      return new Promise((_resolve, reject) => {
+        if (signal.aborted) {
+          const err = new Error('The operation was aborted')
+          err.name = 'AbortError'
+          reject(err)
+          return
+        }
+        signal.addEventListener('abort', () => {
+          const err = new Error('The operation was aborted')
+          err.name = 'AbortError'
+          reject(err)
+        }, { once: true })
+      })
+    })
+
+    const promise = sttApi.transcribe(blob, 'test-user', abortController.signal)
+
+    abortController.abort()
+
+    await expect(promise).rejects.toThrow(FetchError)
+    await expect(promise).rejects.toMatchObject({
+      statusCode: 499,
+      code: 'CANCELED',
+    })
+  })
+
+  it('throws 408 TIMEOUT when transcription times out', async () => {
+    vi.useFakeTimers()
+
+    const blob = new Blob([], { type: 'audio/webm;codecs=opus' })
+
+    const mockFetch = global.fetch as ReturnType<typeof vi.fn>
+    mockFetch.mockImplementation((_url: string, options: RequestInit) => {
+      const signal = options.signal as AbortSignal
+      return new Promise((_resolve, reject) => {
+        if (signal.aborted) {
+          const err = new Error('The operation was aborted')
+          err.name = 'AbortError'
+          reject(err)
+          return
+        }
+        signal.addEventListener('abort', () => {
+          const err = new Error('The operation was aborted')
+          err.name = 'AbortError'
+          reject(err)
+        }, { once: true })
+      })
+    })
+
+    const promise = sttApi.transcribe(blob, 'test-user')
+
+    vi.advanceTimersByTime(60000)
+
+    await expect(promise).rejects.toThrow(FetchError)
+    await expect(promise).rejects.toMatchObject({
+      statusCode: 408,
+      code: 'TIMEOUT',
+    })
+
+    vi.useRealTimers()
+  })
+})

frontend/src/api/stt.test.ts

@@ -54,18 +54,27 @@ export const sttApi = {
     urlObj.searchParams.set('userId', userId)
 
     const controller = new AbortController()
-    const timeoutId = setTimeout(() => controller.abort(), 60000)
+    let timeoutFired = false
+    const timeoutId = setTimeout(() => {
+      timeoutFired = true
+      controller.abort()
+    }, 60000)
 
-    const abortSignal = signal || controller.signal
+    if (signal?.aborted) {
+      controller.abort()
+    }
+    const onAbort = () => controller.abort()
+    signal?.addEventListener('abort', onAbort, { once: true })
 
     try {
       const response = await fetch(urlObj.toString(), {
         method: 'POST',
         body: formData,
-        signal: abortSignal,
+        signal: controller.signal,
       })
 
       clearTimeout(timeoutId)
+      signal?.removeEventListener('abort', onAbort)
 
       if (!response.ok) {
         const data = await response.json().catch(() => ({ error: 'Transcription failed' }))
@@ -75,7 +84,12 @@ export const sttApi = {
       return response.json()
     } catch (error) {
       clearTimeout(timeoutId)
+      signal?.removeEventListener('abort', onAbort)
+
       if (error instanceof Error && error.name === 'AbortError') {
+        if (signal?.aborted && !timeoutFired) {
+          throw new FetchError('Transcription canceled', 499, 'CANCELED')
+        }
         throw new FetchError('Transcription timeout', 408, 'TIMEOUT')
       }
       throw error

frontend/src/api/stt.ts

@@ -47,7 +47,8 @@ const revokeBlobUrls = (attachments: ImageAttachment[]) => {
 
 const ACCEPTED_FILE_TYPES = [...ACCEPTED_IMAGE_TYPES, "application/pdf"]
 
-const VOICE_SEND_SWIPE_THRESHOLD = 30
+const VOICE_SEND_SWIPE_ARM_THRESHOLD = 24
+const VOICE_SEND_SWIPE_DISARM_THRESHOLD = 8
 type VoiceButtonVariant = 'desktop' | 'mobile'
 
 
@@ -496,6 +497,7 @@ export const PromptInput = memo(forwardRef<PromptInputHandle, PromptInputProps>(
       return
     }
 
+    event.preventDefault()
     voiceGestureStartYRef.current = event.clientY
     voiceSwipeArmedRef.current = false
     setIsVoiceSwipeArmed(false)
@@ -510,8 +512,11 @@ export const PromptInput = memo(forwardRef<PromptInputHandle, PromptInputProps>(
       return
     }
 
+    event.preventDefault()
     const deltaY = voiceGestureStartYRef.current - event.clientY
-    const nextIsSwipeArmed = deltaY >= VOICE_SEND_SWIPE_THRESHOLD
+    const nextIsSwipeArmed = voiceSwipeArmedRef.current
+      ? deltaY >= VOICE_SEND_SWIPE_DISARM_THRESHOLD
+      : deltaY >= VOICE_SEND_SWIPE_ARM_THRESHOLD
 
     if (nextIsSwipeArmed !== voiceSwipeArmedRef.current) {
       voiceSwipeArmedRef.current = nextIsSwipeArmed
@@ -532,6 +537,7 @@ export const PromptInput = memo(forwardRef<PromptInputHandle, PromptInputProps>(
       return
     }
 
+    event.preventDefault()
     voiceGestureStartYRef.current = null
 
     if (canceled || !voiceSwipeArmedRef.current) {
@@ -1096,14 +1102,17 @@ if (isIOS && isSecureContext && navigator.clipboard && navigator.clipboard.read)
           : 'bg-muted hover:bg-muted-foreground/20 text-muted-foreground hover:text-foreground border-border active:bg-muted-foreground/30 active:scale-95'
       }`
 
-    const containerClassName = isDesktop ? 'relative hidden md:block' : 'relative flex w-full'
+    const containerClassName = isDesktop ? 'relative hidden md:block' : 'relative flex w-full touch-none select-none'
 
     return (
       <div
         ref={voiceButtonContainerRef}
         className={containerClassName}
         {...voiceGestureHandlers}
       >
+        {!isDesktop && showVoiceFeedback && (
+          <div aria-hidden="true" className="absolute inset-x-0 bottom-full z-20 h-44 touch-none" />
+        )}
         <VoiceStatusOverlay show={!isDesktop && showVoiceFeedback} label={voiceFeedbackLabel} state={voiceFeedbackState} />
         <button
           type="button"

frontend/src/components/message/PromptInput.tsx

@@ -48,7 +48,7 @@ export function VoiceStatusOverlay({ show, label, state }: VoiceStatusOverlayPro
       className="pointer-events-none absolute inset-x-0 bottom-0 z-10"
     >
       <span className="sr-only">{label}</span>
-      <div className="relative flex h-36 w-full flex-col items-center justify-between overflow-hidden rounded-xl border border-green-300/70 bg-gradient-to-t from-green-700 via-green-500 to-emerald-400 px-1 py-3 text-white shadow-lg shadow-green-500/40">
+      <div className="animate-voice-overlay-in relative flex h-44 w-full flex-col items-center justify-between overflow-hidden rounded-xl border border-green-300/70 bg-gradient-to-t from-green-700 via-green-500 to-emerald-400 px-1 py-4 text-white shadow-lg shadow-green-500/40">
         <div className="absolute inset-x-1 top-1 h-10 rounded-full bg-white/20 blur-sm" />
         <div className="relative flex flex-1 flex-col items-center justify-center gap-1">
           {isLoading ? (

frontend/src/components/message/VoiceStatusOverlay.tsx

@@ -127,6 +127,9 @@ export function STTSettings() {
       } else if (sttError) {
         setTestResult('failed')
         setIsTesting(false)
+      } else {
+        setTestResult('idle')
+        setIsTesting(false)
       }
     }
   }, [isTesting, isRecording, isProcessing, transcript, sttError])