diff --git a/ui/src/lib/sceneRecipe.ts b/ui/src/lib/sceneRecipe.ts index 8c184fd2..123ed8f3 100644 --- a/ui/src/lib/sceneRecipe.ts +++ b/ui/src/lib/sceneRecipe.ts @@ -1290,6 +1290,25 @@ export function recipeAssetDuration(recipe: SceneRecipe, assetId: string): numbe return Math.max(0.5, ...(durations.length ? durations : [recipe.scene.duration || 5])) } +/** TTS/SFX/music generation length for one recipe track. + * + * `scene.duration` is only a per-shot fallback (often the first hold). Qwen3 + * and the other audio engines treat `duration_seconds` as a hard cap, so a + * later 9s spoken shot must not inherit an 6–8s scene default or the line is + * truncated while mouth keyframes keep moving. + */ +export function recipeAudioDuration(recipe: SceneRecipe, trackId: string): number { + const shots = listRecipeShots(recipe) + const fromShots = shots + .filter(shot => shot.audioTrackIds === undefined || shot.audioTrackIds.includes(trackId)) + .map(shot => shot.duration || recipe.scene.duration || 5) + const fromBeats = (recipe.dialogueBeats ?? []) + .filter(beat => beat.audioTrackId === trackId) + .map(beat => beat.end) + const fallback = recipe.scene.duration || 5 + return Math.max(0.5, ...(fromShots.length ? fromShots : [fallback]), ...fromBeats) +} + function scopeRecipeToShot(recipe: SceneRecipe, shot: SceneRecipeShot): SceneRecipe { const audioIds = shot.audioTrackIds === undefined ? undefined : new Set(shot.audioTrackIds) const beatIds = shot.dialogueBeatIds === undefined ? undefined : new Set(shot.dialogueBeatIds) diff --git a/ui/src/lib/sceneRecipeAssets.ts b/ui/src/lib/sceneRecipeAssets.ts index 4e0e2d90..2c2b70b5 100644 --- a/ui/src/lib/sceneRecipeAssets.ts +++ b/ui/src/lib/sceneRecipeAssets.ts @@ -1,7 +1,7 @@ import * as api from '../api/client' import { generateImageAsset } from './imageGeneration' import type { SceneRecipe, SceneRecipeAsset, SceneRecipeAudio } from './sceneRecipe' -import { aspectRatioForScene, h3FramesForDuration, h3ResolutionForScene, recipeAssetDuration } from './sceneRecipe' +import { aspectRatioForScene, h3FramesForDuration, h3ResolutionForScene, recipeAssetDuration, recipeAudioDuration } from './sceneRecipe' const wait = (ms: number) => new Promise(resolve => window.setTimeout(resolve, ms)) @@ -210,7 +210,7 @@ async function resolveAudio( if (track.source) return track.source await waitForGpuIdle(onStatus, signal) onStatus?.(`Generating ${track.kind} “${track.id}”…`) - const started = await api.submitGeneration(recipeAudioGenerationParams(track, recipe.scene.duration || 5, workspace)) + const started = await api.submitGeneration(recipeAudioGenerationParams(track, recipeAudioDuration(recipe, track.id), workspace)) const status = await pollUntil( `${track.kind === 'speech' ? 'Voice' : track.kind === 'sfx' ? 'SFX' : 'Music'} “${track.id}”`, () => api.fetchJobStatus(started.job_id), diff --git a/ui/tests/characterKitEpisode.test.mjs b/ui/tests/characterKitEpisode.test.mjs index 66f22405..e6620ff9 100644 --- a/ui/tests/characterKitEpisode.test.mjs +++ b/ui/tests/characterKitEpisode.test.mjs @@ -2,7 +2,7 @@ import assert from 'node:assert/strict' import test from 'node:test' import { characterKitRecipeInventory, createCharacterKit, mountCharacterKitLayers } from '../src/lib/characterKit.ts' import { composeCharacterKitLook, lockFaceRigMouthPlacement } from '../src/lib/characterKitFaceRig.ts' -import { compileRecipeShot, listRecipeShots, parseSceneRecipe } from '../src/lib/sceneRecipe.ts' +import { compileRecipeShot, listRecipeShots, parseSceneRecipe, recipeAudioDuration } from '../src/lib/sceneRecipe.ts' import { evaluateSceneLayer } from '../src/lib/sceneTimeline.ts' const asset = (id, source, reviewState = 'approved', kind = 'overlay') => ({ @@ -129,6 +129,9 @@ test('CharacterKit episode recipe mounts only approved pieces and isolates audio const pointing = compileRecipeShot(recipe, shots[3], {}, filename => filename) assert.ok(pointing.layers.some(layer => layer.id === 'kit-luma-pose-pointing')) assert.deepEqual(pointing.audioTracks.map(track => track.id), ['voice-luma-2']) + assert.equal(recipe.scene.duration, 8) + assert.equal(shots[3].duration, 9) + assert.equal(recipeAudioDuration(recipe, 'voice-luma-2'), 9) }) test('mini South Park-style cutout dialogue locks mouths and scopes speech per shot', () => { diff --git a/ui/tests/sceneRecipe.test.mjs b/ui/tests/sceneRecipe.test.mjs index aa4c90cf..51019731 100644 --- a/ui/tests/sceneRecipe.test.mjs +++ b/ui/tests/sceneRecipe.test.mjs @@ -14,6 +14,7 @@ import { parseSceneRecipe, parseSceneRecipeText, recipeAssetDuration, + recipeAudioDuration, } from '../src/lib/sceneRecipe.ts' test('example saucer recipe compiles to a 4-layer scene with space-cruise motion', () => { @@ -395,6 +396,46 @@ test('H3 plates use supported model canvases and enough temporal-grid frames', ( assert.equal(recipeAssetDuration(recipe, 'clouds'), 9) }) +test('recipe speech uses the spoken shot length, not the shorter scene default', () => { + const layers = [ + { id: 'camera', type: 'camera', cameraPreset: 'camera-locked' }, + { id: 'hero', type: 'image', asset: 'hero-art' }, + { id: 'mouth-wide', name: 'Mouth wide', type: 'overlay', asset: 'mouth-art', faceBinding: { poseLayerId: 'hero', role: 'mouth', state: 'wide' } }, + ] + const recipe = parseSceneRecipe({ + version: 1, + name: 'episode-hold-then-line', + assets: [ + { id: 'hero-art', kind: 'image', source: 'hero.png' }, + { id: 'mouth-art', kind: 'image', source: 'mouth.png' }, + ], + audio: [ + { id: 'voice-snowman', kind: 'speech', prompt: 'El timbre de verdad está detrás del muñeco de nieve.' }, + { id: 'voice-bell', kind: 'speech', prompt: 'La campana del patio está congelada.' }, + ], + dialogueBeats: [ + { id: 'beat-snowman', text: 'El timbre de verdad está detrás del muñeco de nieve.', start: 0.3, end: 7.6, mouthLayerIds: ['mouth-wide'], audioTrackId: 'voice-snowman' }, + { id: 'beat-bell', text: 'La campana del patio está congelada.', start: 0.4, end: 6.8, mouthLayerIds: ['mouth-wide'], audioTrackId: 'voice-bell' }, + ], + shots: [ + { name: 'hold', duration: 6, audioTrackIds: [], dialogueBeatIds: [], layers }, + { name: 'bell', duration: 8, audioTrackIds: ['voice-bell'], dialogueBeatIds: ['beat-bell'], layers }, + { name: 'snowman', duration: 9, audioTrackIds: ['voice-snowman'], dialogueBeatIds: ['beat-snowman'], layers }, + ], + scene: { width: 1280, height: 720, fps: 30, duration: 8, layers }, + }) + assert.equal(recipe.scene.duration, 8) + assert.equal(recipeAudioDuration(recipe, 'voice-snowman'), 9) + assert.equal(recipeAudioDuration(recipe, 'voice-bell'), 8) + + const omittedSceneDuration = parseSceneRecipe({ + ...recipe, + scene: { width: 1280, height: 720, fps: 30, layers }, + }) + assert.equal(omittedSceneDuration.scene.duration, 6) + assert.equal(recipeAudioDuration(omittedSceneDuration, 'voice-snowman'), 9) +}) + const gradedRecipe = grade => parseSceneRecipe({ ...EXAMPLE_SAUCER_CRUISE_RECIPE, scene: { ...EXAMPLE_SAUCER_CRUISE_RECIPE.scene, ...grade }, diff --git a/ui/tests/sceneRecipeAssets.test.mjs b/ui/tests/sceneRecipeAssets.test.mjs index 2a963808..e8bebdaa 100644 --- a/ui/tests/sceneRecipeAssets.test.mjs +++ b/ui/tests/sceneRecipeAssets.test.mjs @@ -1,7 +1,7 @@ import assert from 'node:assert/strict' import test from 'node:test' import { recipeAudioGenerationParams } from '../src/lib/sceneRecipeAssets.ts' -import { withResolvedSources } from '../src/lib/sceneRecipe.ts' +import { parseSceneRecipe, recipeAudioDuration, withResolvedSources } from '../src/lib/sceneRecipe.ts' test('recipe speech uses the proven HocusPocus voice-generation contract', () => { const params = recipeAudioGenerationParams({ @@ -53,3 +53,35 @@ test('recipe audio generation refuses an unresolved silent prompt', () => { /needs a prompt or an existing source/, ) }) + +test('auto-generated episode speech is capped at the spoken shot, not scene.duration', () => { + const layers = [ + { id: 'camera', type: 'camera', cameraPreset: 'camera-locked' }, + { id: 'hero', type: 'image', asset: 'hero-art' }, + { id: 'mouth-wide', name: 'Mouth wide', type: 'overlay', asset: 'mouth-art', faceBinding: { poseLayerId: 'hero', role: 'mouth', state: 'wide' } }, + ] + const recipe = parseSceneRecipe({ + version: 1, + name: 'episode-hold-then-line', + assets: [ + { id: 'hero-art', kind: 'image', source: 'hero.png' }, + { id: 'mouth-art', kind: 'image', source: 'mouth.png' }, + ], + audio: [ + { id: 'voice-snowman', kind: 'speech', prompt: 'El timbre de verdad está detrás del muñeco de nieve.' }, + ], + dialogueBeats: [ + { id: 'beat-snowman', text: 'El timbre de verdad está detrás del muñeco de nieve.', start: 0.3, end: 7.6, mouthLayerIds: ['mouth-wide'], audioTrackId: 'voice-snowman' }, + ], + shots: [ + { name: 'hold', duration: 6, audioTrackIds: [], dialogueBeatIds: [], layers }, + { name: 'snowman', duration: 9, audioTrackIds: ['voice-snowman'], dialogueBeatIds: ['beat-snowman'], layers }, + ], + scene: { width: 1280, height: 720, fps: 30, duration: 8, layers }, + }) + const seconds = recipeAudioDuration(recipe, 'voice-snowman') + assert.equal(seconds, 9) + const params = recipeAudioGenerationParams(recipe.audio[0], seconds, 'episode-room') + assert.equal(params.duration_seconds, 9) + assert.notEqual(params.duration_seconds, recipe.scene.duration) +})