Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions ui/src/lib/sceneRecipe.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1290,6 +1290,25 @@ export function recipeAssetDuration(recipe: SceneRecipe, assetId: string): numbe
return Math.max(0.5, ...(durations.length ? durations : [recipe.scene.duration || 5]))
}

/** TTS/SFX/music generation length for one recipe track.
*
* `scene.duration` is only a per-shot fallback (often the first hold). Qwen3
* and the other audio engines treat `duration_seconds` as a hard cap, so a
* later 9s spoken shot must not inherit an 6–8s scene default or the line is
* truncated while mouth keyframes keep moving.
*/
export function recipeAudioDuration(recipe: SceneRecipe, trackId: string): number {
const shots = listRecipeShots(recipe)
const fromShots = shots
.filter(shot => shot.audioTrackIds === undefined || shot.audioTrackIds.includes(trackId))
.map(shot => shot.duration || recipe.scene.duration || 5)
const fromBeats = (recipe.dialogueBeats ?? [])
.filter(beat => beat.audioTrackId === trackId)
.map(beat => beat.end)
const fallback = recipe.scene.duration || 5
return Math.max(0.5, ...(fromShots.length ? fromShots : [fallback]), ...fromBeats)
}

function scopeRecipeToShot(recipe: SceneRecipe, shot: SceneRecipeShot): SceneRecipe {
const audioIds = shot.audioTrackIds === undefined ? undefined : new Set(shot.audioTrackIds)
const beatIds = shot.dialogueBeatIds === undefined ? undefined : new Set(shot.dialogueBeatIds)
Expand Down
4 changes: 2 additions & 2 deletions ui/src/lib/sceneRecipeAssets.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import * as api from '../api/client'
import { generateImageAsset } from './imageGeneration'
import type { SceneRecipe, SceneRecipeAsset, SceneRecipeAudio } from './sceneRecipe'
import { aspectRatioForScene, h3FramesForDuration, h3ResolutionForScene, recipeAssetDuration } from './sceneRecipe'
import { aspectRatioForScene, h3FramesForDuration, h3ResolutionForScene, recipeAssetDuration, recipeAudioDuration } from './sceneRecipe'

const wait = (ms: number) => new Promise(resolve => window.setTimeout(resolve, ms))

Expand Down Expand Up @@ -210,7 +210,7 @@ async function resolveAudio(
if (track.source) return track.source
await waitForGpuIdle(onStatus, signal)
onStatus?.(`Generating ${track.kind} “${track.id}”…`)
const started = await api.submitGeneration(recipeAudioGenerationParams(track, recipe.scene.duration || 5, workspace))
const started = await api.submitGeneration(recipeAudioGenerationParams(track, recipeAudioDuration(recipe, track.id), workspace))
const status = await pollUntil(
`${track.kind === 'speech' ? 'Voice' : track.kind === 'sfx' ? 'SFX' : 'Music'} “${track.id}”`,
() => api.fetchJobStatus(started.job_id),
Expand Down
5 changes: 4 additions & 1 deletion ui/tests/characterKitEpisode.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import assert from 'node:assert/strict'
import test from 'node:test'
import { characterKitRecipeInventory, createCharacterKit, mountCharacterKitLayers } from '../src/lib/characterKit.ts'
import { composeCharacterKitLook, lockFaceRigMouthPlacement } from '../src/lib/characterKitFaceRig.ts'
import { compileRecipeShot, listRecipeShots, parseSceneRecipe } from '../src/lib/sceneRecipe.ts'
import { compileRecipeShot, listRecipeShots, parseSceneRecipe, recipeAudioDuration } from '../src/lib/sceneRecipe.ts'
import { evaluateSceneLayer } from '../src/lib/sceneTimeline.ts'

const asset = (id, source, reviewState = 'approved', kind = 'overlay') => ({
Expand Down Expand Up @@ -129,6 +129,9 @@ test('CharacterKit episode recipe mounts only approved pieces and isolates audio
const pointing = compileRecipeShot(recipe, shots[3], {}, filename => filename)
assert.ok(pointing.layers.some(layer => layer.id === 'kit-luma-pose-pointing'))
assert.deepEqual(pointing.audioTracks.map(track => track.id), ['voice-luma-2'])
assert.equal(recipe.scene.duration, 8)
assert.equal(shots[3].duration, 9)
assert.equal(recipeAudioDuration(recipe, 'voice-luma-2'), 9)
})

test('mini South Park-style cutout dialogue locks mouths and scopes speech per shot', () => {
Expand Down
41 changes: 41 additions & 0 deletions ui/tests/sceneRecipe.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
parseSceneRecipe,
parseSceneRecipeText,
recipeAssetDuration,
recipeAudioDuration,
} from '../src/lib/sceneRecipe.ts'

test('example saucer recipe compiles to a 4-layer scene with space-cruise motion', () => {
Expand Down Expand Up @@ -395,6 +396,46 @@ test('H3 plates use supported model canvases and enough temporal-grid frames', (
assert.equal(recipeAssetDuration(recipe, 'clouds'), 9)
})

test('recipe speech uses the spoken shot length, not the shorter scene default', () => {
const layers = [
{ id: 'camera', type: 'camera', cameraPreset: 'camera-locked' },
{ id: 'hero', type: 'image', asset: 'hero-art' },
{ id: 'mouth-wide', name: 'Mouth wide', type: 'overlay', asset: 'mouth-art', faceBinding: { poseLayerId: 'hero', role: 'mouth', state: 'wide' } },
]
const recipe = parseSceneRecipe({
version: 1,
name: 'episode-hold-then-line',
assets: [
{ id: 'hero-art', kind: 'image', source: 'hero.png' },
{ id: 'mouth-art', kind: 'image', source: 'mouth.png' },
],
audio: [
{ id: 'voice-snowman', kind: 'speech', prompt: 'El timbre de verdad está detrás del muñeco de nieve.' },
{ id: 'voice-bell', kind: 'speech', prompt: 'La campana del patio está congelada.' },
],
dialogueBeats: [
{ id: 'beat-snowman', text: 'El timbre de verdad está detrás del muñeco de nieve.', start: 0.3, end: 7.6, mouthLayerIds: ['mouth-wide'], audioTrackId: 'voice-snowman' },
{ id: 'beat-bell', text: 'La campana del patio está congelada.', start: 0.4, end: 6.8, mouthLayerIds: ['mouth-wide'], audioTrackId: 'voice-bell' },
],
shots: [
{ name: 'hold', duration: 6, audioTrackIds: [], dialogueBeatIds: [], layers },
{ name: 'bell', duration: 8, audioTrackIds: ['voice-bell'], dialogueBeatIds: ['beat-bell'], layers },
{ name: 'snowman', duration: 9, audioTrackIds: ['voice-snowman'], dialogueBeatIds: ['beat-snowman'], layers },
],
scene: { width: 1280, height: 720, fps: 30, duration: 8, layers },
})
assert.equal(recipe.scene.duration, 8)
assert.equal(recipeAudioDuration(recipe, 'voice-snowman'), 9)
assert.equal(recipeAudioDuration(recipe, 'voice-bell'), 8)

const omittedSceneDuration = parseSceneRecipe({
...recipe,
scene: { width: 1280, height: 720, fps: 30, layers },
})
assert.equal(omittedSceneDuration.scene.duration, 6)
assert.equal(recipeAudioDuration(omittedSceneDuration, 'voice-snowman'), 9)
})

const gradedRecipe = grade => parseSceneRecipe({
...EXAMPLE_SAUCER_CRUISE_RECIPE,
scene: { ...EXAMPLE_SAUCER_CRUISE_RECIPE.scene, ...grade },
Expand Down
34 changes: 33 additions & 1 deletion ui/tests/sceneRecipeAssets.test.mjs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import assert from 'node:assert/strict'
import test from 'node:test'
import { recipeAudioGenerationParams } from '../src/lib/sceneRecipeAssets.ts'
import { withResolvedSources } from '../src/lib/sceneRecipe.ts'
import { parseSceneRecipe, recipeAudioDuration, withResolvedSources } from '../src/lib/sceneRecipe.ts'

test('recipe speech uses the proven HocusPocus voice-generation contract', () => {
const params = recipeAudioGenerationParams({
Expand Down Expand Up @@ -53,3 +53,35 @@ test('recipe audio generation refuses an unresolved silent prompt', () => {
/needs a prompt or an existing source/,
)
})

test('auto-generated episode speech is capped at the spoken shot, not scene.duration', () => {
const layers = [
{ id: 'camera', type: 'camera', cameraPreset: 'camera-locked' },
{ id: 'hero', type: 'image', asset: 'hero-art' },
{ id: 'mouth-wide', name: 'Mouth wide', type: 'overlay', asset: 'mouth-art', faceBinding: { poseLayerId: 'hero', role: 'mouth', state: 'wide' } },
]
const recipe = parseSceneRecipe({
version: 1,
name: 'episode-hold-then-line',
assets: [
{ id: 'hero-art', kind: 'image', source: 'hero.png' },
{ id: 'mouth-art', kind: 'image', source: 'mouth.png' },
],
audio: [
{ id: 'voice-snowman', kind: 'speech', prompt: 'El timbre de verdad está detrás del muñeco de nieve.' },
],
dialogueBeats: [
{ id: 'beat-snowman', text: 'El timbre de verdad está detrás del muñeco de nieve.', start: 0.3, end: 7.6, mouthLayerIds: ['mouth-wide'], audioTrackId: 'voice-snowman' },
],
shots: [
{ name: 'hold', duration: 6, audioTrackIds: [], dialogueBeatIds: [], layers },
{ name: 'snowman', duration: 9, audioTrackIds: ['voice-snowman'], dialogueBeatIds: ['beat-snowman'], layers },
],
scene: { width: 1280, height: 720, fps: 30, duration: 8, layers },
})
const seconds = recipeAudioDuration(recipe, 'voice-snowman')
assert.equal(seconds, 9)
const params = recipeAudioGenerationParams(recipe.audio[0], seconds, 'episode-room')
assert.equal(params.duration_seconds, 9)
assert.notEqual(params.duration_seconds, recipe.scene.duration)
})
Loading