From c889106c0d99afdb741db430e4589268c73d109a Mon Sep 17 00:00:00 2001 From: Shreyas Karnik Date: Mon, 17 Aug 2026 21:34:53 -0700 Subject: [PATCH] fix(tts): honor per-scene speed on ElevenLabs and Gemini (#38) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `TTSEngineOptions.speed` was honored by kokoro, openai, sarvam, mlx-audio and transformers, but silently dropped by ElevenLabs and Gemini — both render server-side with no rate parameter and post-convert via `convertToWav()`, which ignored the option entirely. A manifest asking for `speed: 1.2` got 1x audio with no warning. `convertToWav()` now takes a speed and applies `atempo`, chosen over `asetrate` because it resamples in the time domain and so does not shift pitch. A single `atempo` instance is clamped to 0.5-2.0, so `buildAtempoChain()` splits larger changes across several instances whose product is the requested speed. `guardSpeed()` rejects 0, negatives, NaN and Infinity: the chain divides by each stage factor until the remainder lands in range, so those values never converge and would hang TTS synchronously with no output and no error. Merely extreme values clamp to 0.25-4.0 with a warning instead. The clip cache key already includes speed, so cached clips do not go stale. Reported-by: @salir-admin --- src/tts/engine.ts | 74 ++++++++++++++++++++- src/tts/engines/elevenlabs.ts | 5 +- src/tts/engines/gemini.ts | 5 +- tests/tts/speed.test.ts | 121 ++++++++++++++++++++++++++++++++++ 4 files changed, 200 insertions(+), 5 deletions(-) create mode 100644 tests/tts/speed.test.ts diff --git a/src/tts/engine.ts b/src/tts/engine.ts index e865eb1..f7db023 100644 --- a/src/tts/engine.ts +++ b/src/tts/engine.ts @@ -199,14 +199,86 @@ export function parseWavHeader(wav: Buffer): WavHeader { }; } +/** ffmpeg's `atempo` filter accepts a factor in [0.5, 2.0] per instance and + * errors outside it, so a larger change has to be split across several + * instances whose product is the requested speed. */ +const ATEMPO_MIN = 0.5; +const ATEMPO_MAX = 2; + +/** Widest range the chain will honour, i.e. two `atempo` stages either way. + * Past this the voice stops being intelligible, so a request that far out is + * far more likely a typo than an intent. */ +const SPEED_MIN = ATEMPO_MIN * ATEMPO_MIN; +const SPEED_MAX = ATEMPO_MAX * ATEMPO_MAX; + +/** + * Normalise a requested `speed` before it reaches the atempo chain. + * + * Throws on values the chain cannot converge on: it divides by the stage + * factor each pass, so `0`, a negative, or `Infinity` would loop forever — + * a hang mid-TTS with no output and no error. `NaN` escapes both loops and + * emits a literal `atempo=NaN`, which makes ffmpeg exit non-zero inside + * `execFileSync`. Both are misconfiguration, so they fail loudly. + * + * Merely extreme values are clamped rather than rejected — the intent is + * legible even when the number is silly. + */ +function guardSpeed(speed: number): number { + if (!Number.isFinite(speed) || speed <= 0) { + throw new Error(`TTS speed must be a positive finite number, got ${speed}`); + } + const capped = Math.min(SPEED_MAX, Math.max(SPEED_MIN, speed)); + if (capped !== speed) { + console.warn(`Warning: TTS speed ${speed} clamped to ${capped}`); + } + return capped; +} + +/** Trim floating-point noise so the filter string stays readable and ffmpeg + * never sees something like `atempo=1.7999999999999998`. */ +function fmt(n: number): string { + return String(Number(n.toFixed(6))); +} + +/** + * Build the ffmpeg args that change playback rate to `speed`. + * + * `atempo` rather than `asetrate`: it resamples in the time domain, so the + * voice speeds up without shifting pitch. Returns an empty array at 1x so the + * default path spawns ffmpeg with no filter at all. + */ +export function buildAtempoChain(speed: number): string[] { + if (speed === 1) return []; + speed = guardSpeed(speed); + + const stages: number[] = []; + let remaining = speed; + while (remaining > ATEMPO_MAX) { + stages.push(ATEMPO_MAX); + remaining /= ATEMPO_MAX; + } + while (remaining < ATEMPO_MIN) { + stages.push(ATEMPO_MIN); + remaining /= ATEMPO_MIN; + } + stages.push(remaining); + + return ['-filter:a', stages.map(s => `atempo=${fmt(s)}`).join(',')]; +} + /** * Convert arbitrary audio (MP3, OGG, PCM, etc.) to Argo's WAV format * (mono, Float32, 24kHz) using ffmpeg. + * + * `speed` is applied here because engines that render server-side (ElevenLabs, + * Gemini) have no native rate control — this conversion is the only place the + * rate can change. Engines with their own speed parameter must not use it. */ -export function convertToWav(audioBuffer: Buffer): Buffer { +export function convertToWav(audioBuffer: Buffer, speed = 1): Buffer { const { execFileSync } = childProcess; const result = execFileSync('ffmpeg', [ '-i', 'pipe:0', + ...buildAtempoChain(speed), '-f', 'wav', '-acodec', 'pcm_f32le', '-ac', '1', diff --git a/src/tts/engines/elevenlabs.ts b/src/tts/engines/elevenlabs.ts index c4f36c1..2887b96 100644 --- a/src/tts/engines/elevenlabs.ts +++ b/src/tts/engines/elevenlabs.ts @@ -69,8 +69,9 @@ export class ElevenLabsEngine implements TTSEngine { } const mp3Buffer = Buffer.concat(chunks); - // Convert MP3 to Argo WAV format + // Convert MP3 to Argo WAV format. ElevenLabs has no speed parameter, so + // the rate change rides along with the conversion. const { convertToWav } = await import('../engine.js'); - return convertToWav(mp3Buffer); + return convertToWav(mp3Buffer, options.speed ?? 1); } } diff --git a/src/tts/engines/gemini.ts b/src/tts/engines/gemini.ts index 730835a..05ae400 100644 --- a/src/tts/engines/gemini.ts +++ b/src/tts/engines/gemini.ts @@ -70,8 +70,9 @@ export class GeminiEngine implements TTSEngine { const audioBuffer = Buffer.from(audioPart.inlineData.data, 'base64'); - // Convert to Argo WAV format + // Convert to Argo WAV format. Gemini has no speed parameter, so the rate + // change rides along with the conversion. const { convertToWav } = await import('../engine.js'); - return convertToWav(audioBuffer); + return convertToWav(audioBuffer, options.speed ?? 1); } } diff --git a/tests/tts/speed.test.ts b/tests/tts/speed.test.ts new file mode 100644 index 0000000..8b7fbc6 --- /dev/null +++ b/tests/tts/speed.test.ts @@ -0,0 +1,121 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { buildAtempoChain, convertToWav } from '../../src/tts/engine.js'; +import { execFileSync } from 'node:child_process'; + +// `engine.ts` reads `execFileSync` off the namespace at call time, so the whole +// module has to be replaced — an ESM namespace object cannot be spied on. +vi.mock('node:child_process', async importOriginal => ({ + ...(await importOriginal()), + execFileSync: vi.fn(() => Buffer.from('fake-wav')), +})); + +/** + * Regression coverage for #38: `TTSEngineOptions.speed` was silently dropped by + * every engine that renders audio server-side and post-converts with ffmpeg + * (ElevenLabs, Gemini). Those engines have no native speed control, so the rate + * change has to happen during the WAV conversion. + * + * `atempo` is the right filter rather than `asetrate`: it resamples in the time + * domain, so speeding up does not raise the pitch. Its cost is a hard 0.5–2.0 + * clamp per instance, which is why anything outside that range must be split + * across several chained instances whose product is the requested speed. + */ +describe('buildAtempoChain', () => { + /** Multiply the `atempo=N` values out of a chain back into one speed. */ + function effectiveSpeed(args: string[]): number { + const filter = args[args.indexOf('-filter:a') + 1]; + return filter + .split(',') + .map(stage => Number(stage.replace('atempo=', ''))) + .reduce((a, b) => a * b, 1); + } + + it('emits no filter at all for unchanged speed', () => { + expect(buildAtempoChain(1)).toEqual([]); + }); + + it('emits a single stage for a speed-up inside atempo range', () => { + expect(buildAtempoChain(1.5)).toEqual(['-filter:a', 'atempo=1.5']); + }); + + it('emits a single stage for a slow-down inside atempo range', () => { + expect(buildAtempoChain(0.75)).toEqual(['-filter:a', 'atempo=0.75']); + }); + + it('chains stages for a speed-up above the 2.0 per-instance ceiling', () => { + const args = buildAtempoChain(3); + expect(effectiveSpeed(args)).toBeCloseTo(3, 5); + }); + + it('chains stages for a slow-down below the 0.5 per-instance floor', () => { + const args = buildAtempoChain(0.25); + expect(effectiveSpeed(args)).toBeCloseTo(0.25, 5); + }); + + it('keeps every individual stage within ffmpeg\'s 0.5-2.0 limit', () => { + for (const speed of [0.25, 0.3, 0.5, 0.9, 1.1, 2, 3, 4]) { + const args = buildAtempoChain(speed); + const stages = args[1].split(',').map(s => Number(s.replace('atempo=', ''))); + for (const stage of stages) { + expect(stage).toBeGreaterThanOrEqual(0.5); + expect(stage).toBeLessThanOrEqual(2); + } + } + }); +}); + +describe('buildAtempoChain input guard', () => { + // The chain divides by each stage factor until the remainder lands in range, + // so these values never converge and would hang TTS rather than fail it. + it.each([0, -1, NaN, Infinity])('rejects %p instead of looping forever', speed => { + expect(() => buildAtempoChain(speed)).toThrow(/positive finite number/); + }); + + it('names the offending value in the error', () => { + expect(() => buildAtempoChain(0)).toThrow('TTS speed must be a positive finite number, got 0'); + }); + + it('caps an absurd speed-up at 4x and warns', () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const args = buildAtempoChain(100); + expect(args).toEqual(['-filter:a', 'atempo=2,atempo=2']); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('clamped to 4')); + warn.mockRestore(); + }); + + it('caps an absurd slow-down at 0.25x and warns', () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const args = buildAtempoChain(0.01); + expect(args).toEqual(['-filter:a', 'atempo=0.5,atempo=0.5']); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('clamped to 0.25')); + warn.mockRestore(); + }); + + it('leaves an in-range speed untouched and stays quiet', () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + expect(buildAtempoChain(1.5)).toEqual(['-filter:a', 'atempo=1.5']); + expect(warn).not.toHaveBeenCalled(); + warn.mockRestore(); + }); +}); + +describe('convertToWav speed', () => { + const execSpy = vi.mocked(execFileSync); + + beforeEach(() => { + execSpy.mockClear(); + }); + + it('passes no audio filter when speed is left at the default', () => { + convertToWav(Buffer.from('audio')); + const args = execSpy.mock.calls[0][1] as string[]; + expect(args).not.toContain('-filter:a'); + }); + + it('passes the atempo chain to ffmpeg when a speed is requested', () => { + convertToWav(Buffer.from('audio'), 1.25); + const args = execSpy.mock.calls[0][1] as string[]; + expect(args).toContain('-filter:a'); + expect(args[args.indexOf('-filter:a') + 1]).toBe('atempo=1.25'); + }); +});