From 5e43a486a2cb240de5e7d1fc76d583bc51cee4dd Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 12 Sep 2026 15:09:15 +0000 Subject: [PATCH] fix(cut-paper): keep aligned mouths closed until the first word A zero-length closed viseme shared a timestamp with the first spoken word. rebuild kept the later open frame, so evaluateSceneLayer held that viseme from t=0 (Nilo open for 6s, Berta for 17s). Rest closed now starts at t=0. Detect-from-audio no longer applies a dummy plan when every Whisper unit is empty. Co-authored-by: ignaciodelcano+dcl --- .../shots/02-talk.maestro-scene.json | 88 ++++++++++++- .../shots/03-sticker.maestro-scene.json | 44 ++++++- .../shots/en/02-talk.maestro-scene.json | 88 ++++++++++++- .../shots/en/03-sticker.maestro-scene.json | 44 ++++++- .../en/tijeral-la-fuente.maestro-scene.json | 120 ++++++++++++++++++ .../tijeral-la-fuente.maestro-scene.json | 120 ++++++++++++++++++ .../components/Sidebar/SceneAnimatorPanel.tsx | 3 +- ui/src/lib/cutoutDialogue.ts | 12 +- ui/tests/cutPaper.test.ts | 11 +- ui/tests/cutoutDialogue.test.mjs | 27 ++++ 10 files changed, 540 insertions(+), 17 deletions(-) diff --git a/ui/public/examples/cut-paper/shots/02-talk.maestro-scene.json b/ui/public/examples/cut-paper/shots/02-talk.maestro-scene.json index c940b13a..848e9c86 100644 --- a/ui/public/examples/cut-paper/shots/02-talk.maestro-scene.json +++ b/ui/public/examples/cut-paper/shots/02-talk.maestro-scene.json @@ -209,7 +209,7 @@ "x": 36.3612, "y": 54.629, "scale": 0.05039999999999999, - "opacity": 0, + "opacity": 1, "rotation": 0 }, "end": { @@ -222,6 +222,16 @@ "duration": 40, "curve": "hold", "keyframes": [ + { + "id": "puppet-nilo-mouth-closed-dialogue-0", + "time": 0, + "x": 36.3612, + "y": 54.629, + "scale": 0.05039999999999999, + "opacity": 1, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-nilo-mouth-closed-dialogue-6000", "time": 6, @@ -447,6 +457,16 @@ "duration": 40, "curve": "hold", "keyframes": [ + { + "id": "puppet-nilo-mouth-small-dialogue-0", + "time": 0, + "x": 36.3612, + "y": 54.629, + "scale": 0.05039999999999999, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-nilo-mouth-small-dialogue-6000", "time": 6, @@ -659,7 +679,7 @@ "x": 36.3612, "y": 54.629, "scale": 0.05039999999999999, - "opacity": 1, + "opacity": 0, "rotation": 0 }, "end": { @@ -672,6 +692,16 @@ "duration": 40, "curve": "hold", "keyframes": [ + { + "id": "puppet-nilo-mouth-wide-dialogue-0", + "time": 0, + "x": 36.3612, + "y": 54.629, + "scale": 0.05039999999999999, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-nilo-mouth-wide-dialogue-6000", "time": 6, @@ -897,6 +927,16 @@ "duration": 40, "curve": "hold", "keyframes": [ + { + "id": "puppet-nilo-mouth-round-dialogue-0", + "time": 0, + "x": 36.3612, + "y": 54.629, + "scale": 0.05039999999999999, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-nilo-mouth-round-dialogue-6000", "time": 6, @@ -1178,7 +1218,7 @@ "x": 62.0646, "y": 57.711, "scale": 0.041800000000000004, - "opacity": 0, + "opacity": 1, "rotation": 0 }, "end": { @@ -1191,6 +1231,16 @@ "duration": 40, "curve": "hold", "keyframes": [ + { + "id": "puppet-berta-mouth-closed-dialogue-0", + "time": 0, + "x": 62.0646, + "y": 57.711, + "scale": 0.041800000000000004, + "opacity": 1, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-berta-mouth-closed-dialogue-17000", "time": 17, @@ -1386,6 +1436,16 @@ "duration": 40, "curve": "hold", "keyframes": [ + { + "id": "puppet-berta-mouth-small-dialogue-0", + "time": 0, + "x": 62.0646, + "y": 57.711, + "scale": 0.041800000000000004, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-berta-mouth-small-dialogue-17000", "time": 17, @@ -1568,7 +1628,7 @@ "x": 62.0646, "y": 57.711, "scale": 0.041800000000000004, - "opacity": 1, + "opacity": 0, "rotation": 0 }, "end": { @@ -1581,6 +1641,16 @@ "duration": 40, "curve": "hold", "keyframes": [ + { + "id": "puppet-berta-mouth-wide-dialogue-0", + "time": 0, + "x": 62.0646, + "y": 57.711, + "scale": 0.041800000000000004, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-berta-mouth-wide-dialogue-17000", "time": 17, @@ -1776,6 +1846,16 @@ "duration": 40, "curve": "hold", "keyframes": [ + { + "id": "puppet-berta-mouth-round-dialogue-0", + "time": 0, + "x": 62.0646, + "y": 57.711, + "scale": 0.041800000000000004, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-berta-mouth-round-dialogue-17000", "time": 17, diff --git a/ui/public/examples/cut-paper/shots/03-sticker.maestro-scene.json b/ui/public/examples/cut-paper/shots/03-sticker.maestro-scene.json index 273a194f..fcf28464 100644 --- a/ui/public/examples/cut-paper/shots/03-sticker.maestro-scene.json +++ b/ui/public/examples/cut-paper/shots/03-sticker.maestro-scene.json @@ -887,7 +887,7 @@ "x": 118, "y": 71.43136, "scale": 0.03584, - "opacity": 0, + "opacity": 1, "rotation": 0 }, "end": { @@ -900,6 +900,16 @@ "duration": 26, "curve": "hold", "keyframes": [ + { + "id": "puppet-kito-mouth-closed-dialogue-0", + "time": 0, + "x": 118, + "y": 71.43136, + "scale": 0.03584, + "opacity": 1, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-kito-mouth-closed-dialogue-10000", "time": 10, @@ -995,6 +1005,16 @@ "duration": 26, "curve": "hold", "keyframes": [ + { + "id": "puppet-kito-mouth-small-dialogue-0", + "time": 0, + "x": 118, + "y": 71.43136, + "scale": 0.03584, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-kito-mouth-small-dialogue-10000", "time": 10, @@ -1077,7 +1097,7 @@ "x": 118, "y": 71.43136, "scale": 0.03584, - "opacity": 1, + "opacity": 0, "rotation": 0 }, "end": { @@ -1090,6 +1110,16 @@ "duration": 26, "curve": "hold", "keyframes": [ + { + "id": "puppet-kito-mouth-wide-dialogue-0", + "time": 0, + "x": 118, + "y": 71.43136, + "scale": 0.03584, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-kito-mouth-wide-dialogue-10000", "time": 10, @@ -1185,6 +1215,16 @@ "duration": 26, "curve": "hold", "keyframes": [ + { + "id": "puppet-kito-mouth-round-dialogue-0", + "time": 0, + "x": 118, + "y": 71.43136, + "scale": 0.03584, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-kito-mouth-round-dialogue-10000", "time": 10, diff --git a/ui/public/examples/cut-paper/shots/en/02-talk.maestro-scene.json b/ui/public/examples/cut-paper/shots/en/02-talk.maestro-scene.json index e1e6c9e7..43b718ce 100644 --- a/ui/public/examples/cut-paper/shots/en/02-talk.maestro-scene.json +++ b/ui/public/examples/cut-paper/shots/en/02-talk.maestro-scene.json @@ -209,7 +209,7 @@ "x": 36.3612, "y": 54.629, "scale": 0.05039999999999999, - "opacity": 0, + "opacity": 1, "rotation": 0 }, "end": { @@ -222,6 +222,16 @@ "duration": 40, "curve": "hold", "keyframes": [ + { + "id": "puppet-nilo-mouth-closed-dialogue-0", + "time": 0, + "x": 36.3612, + "y": 54.629, + "scale": 0.05039999999999999, + "opacity": 1, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-nilo-mouth-closed-dialogue-6000", "time": 6, @@ -487,6 +497,16 @@ "duration": 40, "curve": "hold", "keyframes": [ + { + "id": "puppet-nilo-mouth-small-dialogue-0", + "time": 0, + "x": 36.3612, + "y": 54.629, + "scale": 0.05039999999999999, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-nilo-mouth-small-dialogue-6000", "time": 6, @@ -739,7 +759,7 @@ "x": 36.3612, "y": 54.629, "scale": 0.05039999999999999, - "opacity": 1, + "opacity": 0, "rotation": 0 }, "end": { @@ -752,6 +772,16 @@ "duration": 40, "curve": "hold", "keyframes": [ + { + "id": "puppet-nilo-mouth-wide-dialogue-0", + "time": 0, + "x": 36.3612, + "y": 54.629, + "scale": 0.05039999999999999, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-nilo-mouth-wide-dialogue-6000", "time": 6, @@ -1017,6 +1047,16 @@ "duration": 40, "curve": "hold", "keyframes": [ + { + "id": "puppet-nilo-mouth-round-dialogue-0", + "time": 0, + "x": 36.3612, + "y": 54.629, + "scale": 0.05039999999999999, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-nilo-mouth-round-dialogue-6000", "time": 6, @@ -1338,7 +1378,7 @@ "x": 62.0646, "y": 57.711, "scale": 0.041800000000000004, - "opacity": 0, + "opacity": 1, "rotation": 0 }, "end": { @@ -1351,6 +1391,16 @@ "duration": 40, "curve": "hold", "keyframes": [ + { + "id": "puppet-berta-mouth-closed-dialogue-0", + "time": 0, + "x": 62.0646, + "y": 57.711, + "scale": 0.041800000000000004, + "opacity": 1, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-berta-mouth-closed-dialogue-17000", "time": 17, @@ -1566,6 +1616,16 @@ "duration": 40, "curve": "hold", "keyframes": [ + { + "id": "puppet-berta-mouth-small-dialogue-0", + "time": 0, + "x": 62.0646, + "y": 57.711, + "scale": 0.041800000000000004, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-berta-mouth-small-dialogue-17000", "time": 17, @@ -1768,7 +1828,7 @@ "x": 62.0646, "y": 57.711, "scale": 0.041800000000000004, - "opacity": 1, + "opacity": 0, "rotation": 0 }, "end": { @@ -1781,6 +1841,16 @@ "duration": 40, "curve": "hold", "keyframes": [ + { + "id": "puppet-berta-mouth-wide-dialogue-0", + "time": 0, + "x": 62.0646, + "y": 57.711, + "scale": 0.041800000000000004, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-berta-mouth-wide-dialogue-17000", "time": 17, @@ -1996,6 +2066,16 @@ "duration": 40, "curve": "hold", "keyframes": [ + { + "id": "puppet-berta-mouth-round-dialogue-0", + "time": 0, + "x": 62.0646, + "y": 57.711, + "scale": 0.041800000000000004, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-berta-mouth-round-dialogue-17000", "time": 17, diff --git a/ui/public/examples/cut-paper/shots/en/03-sticker.maestro-scene.json b/ui/public/examples/cut-paper/shots/en/03-sticker.maestro-scene.json index 725f285d..138299ec 100644 --- a/ui/public/examples/cut-paper/shots/en/03-sticker.maestro-scene.json +++ b/ui/public/examples/cut-paper/shots/en/03-sticker.maestro-scene.json @@ -887,7 +887,7 @@ "x": 118, "y": 71.43136, "scale": 0.03584, - "opacity": 0, + "opacity": 1, "rotation": 0 }, "end": { @@ -900,6 +900,16 @@ "duration": 26, "curve": "hold", "keyframes": [ + { + "id": "puppet-kito-mouth-closed-dialogue-0", + "time": 0, + "x": 118, + "y": 71.43136, + "scale": 0.03584, + "opacity": 1, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-kito-mouth-closed-dialogue-10000", "time": 10, @@ -992,7 +1002,7 @@ "x": 118, "y": 71.43136, "scale": 0.03584, - "opacity": 1, + "opacity": 0, "rotation": 0 }, "end": { @@ -1005,6 +1015,16 @@ "duration": 26, "curve": "hold", "keyframes": [ + { + "id": "puppet-kito-mouth-small-dialogue-0", + "time": 0, + "x": 118, + "y": 71.43136, + "scale": 0.03584, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-kito-mouth-small-dialogue-10000", "time": 10, @@ -1110,6 +1130,16 @@ "duration": 26, "curve": "hold", "keyframes": [ + { + "id": "puppet-kito-mouth-wide-dialogue-0", + "time": 0, + "x": 118, + "y": 71.43136, + "scale": 0.03584, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-kito-mouth-wide-dialogue-10000", "time": 10, @@ -1215,6 +1245,16 @@ "duration": 26, "curve": "hold", "keyframes": [ + { + "id": "puppet-kito-mouth-round-dialogue-0", + "time": 0, + "x": 118, + "y": 71.43136, + "scale": 0.03584, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-kito-mouth-round-dialogue-10000", "time": 10, diff --git a/ui/public/examples/cut-paper/shots/en/tijeral-la-fuente.maestro-scene.json b/ui/public/examples/cut-paper/shots/en/tijeral-la-fuente.maestro-scene.json index 27ffa035..d5c51ba7 100644 --- a/ui/public/examples/cut-paper/shots/en/tijeral-la-fuente.maestro-scene.json +++ b/ui/public/examples/cut-paper/shots/en/tijeral-la-fuente.maestro-scene.json @@ -222,6 +222,16 @@ "duration": 78, "curve": "hold", "keyframes": [ + { + "id": "puppet-nilo-mouth-closed-dialogue-0", + "time": 0, + "x": 36.3612, + "y": 54.629, + "scale": 0.05039999999999999, + "opacity": 1, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-nilo-mouth-closed-dialogue-6000", "time": 6, @@ -477,6 +487,16 @@ "duration": 78, "curve": "hold", "keyframes": [ + { + "id": "puppet-nilo-mouth-small-dialogue-0", + "time": 0, + "x": 36.3612, + "y": 54.629, + "scale": 0.05039999999999999, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-nilo-mouth-small-dialogue-6000", "time": 6, @@ -732,6 +752,16 @@ "duration": 78, "curve": "hold", "keyframes": [ + { + "id": "puppet-nilo-mouth-wide-dialogue-0", + "time": 0, + "x": 36.3612, + "y": 54.629, + "scale": 0.05039999999999999, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-nilo-mouth-wide-dialogue-6000", "time": 6, @@ -987,6 +1017,16 @@ "duration": 78, "curve": "hold", "keyframes": [ + { + "id": "puppet-nilo-mouth-round-dialogue-0", + "time": 0, + "x": 36.3612, + "y": 54.629, + "scale": 0.05039999999999999, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-nilo-mouth-round-dialogue-6000", "time": 6, @@ -1311,6 +1351,16 @@ "duration": 78, "curve": "hold", "keyframes": [ + { + "id": "puppet-berta-mouth-closed-dialogue-0", + "time": 0, + "x": 62.0646, + "y": 57.711, + "scale": 0.041800000000000004, + "opacity": 1, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-berta-mouth-closed-dialogue-17000", "time": 17, @@ -1516,6 +1566,16 @@ "duration": 78, "curve": "hold", "keyframes": [ + { + "id": "puppet-berta-mouth-small-dialogue-0", + "time": 0, + "x": 62.0646, + "y": 57.711, + "scale": 0.041800000000000004, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-berta-mouth-small-dialogue-17000", "time": 17, @@ -1721,6 +1781,16 @@ "duration": 78, "curve": "hold", "keyframes": [ + { + "id": "puppet-berta-mouth-wide-dialogue-0", + "time": 0, + "x": 62.0646, + "y": 57.711, + "scale": 0.041800000000000004, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-berta-mouth-wide-dialogue-17000", "time": 17, @@ -1926,6 +1996,16 @@ "duration": 78, "curve": "hold", "keyframes": [ + { + "id": "puppet-berta-mouth-round-dialogue-0", + "time": 0, + "x": 62.0646, + "y": 57.711, + "scale": 0.041800000000000004, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-berta-mouth-round-dialogue-17000", "time": 17, @@ -2218,6 +2298,16 @@ "duration": 78, "curve": "hold", "keyframes": [ + { + "id": "puppet-kito-mouth-closed-dialogue-0", + "time": 0, + "x": 118, + "y": 71.43136, + "scale": 0.03584, + "opacity": 1, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-kito-mouth-closed-dialogue-62000", "time": 62, @@ -2313,6 +2403,16 @@ "duration": 78, "curve": "hold", "keyframes": [ + { + "id": "puppet-kito-mouth-small-dialogue-0", + "time": 0, + "x": 118, + "y": 71.43136, + "scale": 0.03584, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-kito-mouth-small-dialogue-62000", "time": 62, @@ -2408,6 +2508,16 @@ "duration": 78, "curve": "hold", "keyframes": [ + { + "id": "puppet-kito-mouth-wide-dialogue-0", + "time": 0, + "x": 118, + "y": 71.43136, + "scale": 0.03584, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-kito-mouth-wide-dialogue-62000", "time": 62, @@ -2503,6 +2613,16 @@ "duration": 78, "curve": "hold", "keyframes": [ + { + "id": "puppet-kito-mouth-round-dialogue-0", + "time": 0, + "x": 118, + "y": 71.43136, + "scale": 0.03584, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-kito-mouth-round-dialogue-62000", "time": 62, diff --git a/ui/public/examples/cut-paper/tijeral-la-fuente.maestro-scene.json b/ui/public/examples/cut-paper/tijeral-la-fuente.maestro-scene.json index fb345386..a71a79d0 100644 --- a/ui/public/examples/cut-paper/tijeral-la-fuente.maestro-scene.json +++ b/ui/public/examples/cut-paper/tijeral-la-fuente.maestro-scene.json @@ -222,6 +222,16 @@ "duration": 78, "curve": "hold", "keyframes": [ + { + "id": "puppet-nilo-mouth-closed-dialogue-0", + "time": 0, + "x": 36.3612, + "y": 54.629, + "scale": 0.05039999999999999, + "opacity": 1, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-nilo-mouth-closed-dialogue-6000", "time": 6, @@ -437,6 +447,16 @@ "duration": 78, "curve": "hold", "keyframes": [ + { + "id": "puppet-nilo-mouth-small-dialogue-0", + "time": 0, + "x": 36.3612, + "y": 54.629, + "scale": 0.05039999999999999, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-nilo-mouth-small-dialogue-6000", "time": 6, @@ -652,6 +672,16 @@ "duration": 78, "curve": "hold", "keyframes": [ + { + "id": "puppet-nilo-mouth-wide-dialogue-0", + "time": 0, + "x": 36.3612, + "y": 54.629, + "scale": 0.05039999999999999, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-nilo-mouth-wide-dialogue-6000", "time": 6, @@ -867,6 +897,16 @@ "duration": 78, "curve": "hold", "keyframes": [ + { + "id": "puppet-nilo-mouth-round-dialogue-0", + "time": 0, + "x": 36.3612, + "y": 54.629, + "scale": 0.05039999999999999, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-nilo-mouth-round-dialogue-6000", "time": 6, @@ -1151,6 +1191,16 @@ "duration": 78, "curve": "hold", "keyframes": [ + { + "id": "puppet-berta-mouth-closed-dialogue-0", + "time": 0, + "x": 62.0646, + "y": 57.711, + "scale": 0.041800000000000004, + "opacity": 1, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-berta-mouth-closed-dialogue-17000", "time": 17, @@ -1336,6 +1386,16 @@ "duration": 78, "curve": "hold", "keyframes": [ + { + "id": "puppet-berta-mouth-small-dialogue-0", + "time": 0, + "x": 62.0646, + "y": 57.711, + "scale": 0.041800000000000004, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-berta-mouth-small-dialogue-17000", "time": 17, @@ -1521,6 +1581,16 @@ "duration": 78, "curve": "hold", "keyframes": [ + { + "id": "puppet-berta-mouth-wide-dialogue-0", + "time": 0, + "x": 62.0646, + "y": 57.711, + "scale": 0.041800000000000004, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-berta-mouth-wide-dialogue-17000", "time": 17, @@ -1706,6 +1776,16 @@ "duration": 78, "curve": "hold", "keyframes": [ + { + "id": "puppet-berta-mouth-round-dialogue-0", + "time": 0, + "x": 62.0646, + "y": 57.711, + "scale": 0.041800000000000004, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-berta-mouth-round-dialogue-17000", "time": 17, @@ -1978,6 +2058,16 @@ "duration": 78, "curve": "hold", "keyframes": [ + { + "id": "puppet-kito-mouth-closed-dialogue-0", + "time": 0, + "x": 118, + "y": 71.43136, + "scale": 0.03584, + "opacity": 1, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-kito-mouth-closed-dialogue-62000", "time": 62, @@ -2063,6 +2153,16 @@ "duration": 78, "curve": "hold", "keyframes": [ + { + "id": "puppet-kito-mouth-small-dialogue-0", + "time": 0, + "x": 118, + "y": 71.43136, + "scale": 0.03584, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-kito-mouth-small-dialogue-62000", "time": 62, @@ -2148,6 +2248,16 @@ "duration": 78, "curve": "hold", "keyframes": [ + { + "id": "puppet-kito-mouth-wide-dialogue-0", + "time": 0, + "x": 118, + "y": 71.43136, + "scale": 0.03584, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-kito-mouth-wide-dialogue-62000", "time": 62, @@ -2233,6 +2343,16 @@ "duration": 78, "curve": "hold", "keyframes": [ + { + "id": "puppet-kito-mouth-round-dialogue-0", + "time": 0, + "x": 118, + "y": 71.43136, + "scale": 0.03584, + "opacity": 0, + "rotation": 0, + "curve": "hold" + }, { "id": "puppet-kito-mouth-round-dialogue-62000", "time": 62, diff --git a/ui/src/components/Sidebar/SceneAnimatorPanel.tsx b/ui/src/components/Sidebar/SceneAnimatorPanel.tsx index a2689e6b..7b179f4b 100644 --- a/ui/src/components/Sidebar/SceneAnimatorPanel.tsx +++ b/ui/src/components/Sidebar/SceneAnimatorPanel.tsx @@ -2875,7 +2875,8 @@ export function SceneAnimatorPanel() { const units = segments.flatMap(segment => segment.words?.length ? segment.words.map(word => ({ text: word.text, start: word.start, end: word.end })) : [{ text: segment.text, start: segment.start, end: segment.end }]) - .filter(unit => unit.end > unit.start && unit.start + track.startTime < scene.duration) + .filter(unit => unit.text.trim() && unit.end > unit.start && unit.start + track.startTime < scene.duration) + if (!units.length) throw new Error('No spoken regions were found in this track.') const plan = planAlignedCutoutDialogue(units.map(unit => ({ text: unit.text, start: Math.max(0, unit.start + track.startTime), diff --git a/ui/src/lib/cutoutDialogue.ts b/ui/src/lib/cutoutDialogue.ts index 5295e285..261fba58 100644 --- a/ui/src/lib/cutoutDialogue.ts +++ b/ui/src/lib/cutoutDialogue.ts @@ -229,17 +229,23 @@ export function planAlignedCutoutDialogue( const frame = 1 / Math.max(1, fps) const start = Math.max(0, usable[0].start) const end = Math.max(start + frame, usable[usable.length - 1].end) - const visemes: CutoutDialoguePlan['visemes'] = [{ start, end: start, state: 'closed' }] + // Rest closed must occupy a distinct time before the first spoken viseme. + // A zero-length closed at `start` shares a timestamp with the first word; + // rebuild keeps the later (open) frame, so evaluateSceneLayer holds that + // open mouth from t=0 until speech begins. + const visemes: CutoutDialoguePlan['visemes'] = start > 0 + ? [{ start: 0, end: start, state: 'closed' }] + : [] let cursor = start for (const unit of usable) { const unitStart = Math.max(start, unit.start) const unitEnd = Math.max(unitStart + frame, unit.end) - if (unitStart > cursor + frame) visemes.push({ start: cursor, end: unitStart, state: 'closed' }) + if (unitStart > cursor) visemes.push({ start: cursor, end: unitStart, state: 'closed' }) const spoken = visemeForToken(unit.text) visemes.push({ start: unitStart, end: unitEnd, state: spoken === 'closed' ? 'small' : spoken }) cursor = unitEnd } - visemes.push({ start: cursor, end, state: 'closed' }) + visemes.push({ start: cursor, end: Math.max(cursor, end), state: 'closed' }) return { start, end, visemes } } diff --git a/ui/tests/cutPaper.test.ts b/ui/tests/cutPaper.test.ts index 632755a3..541cfb31 100644 --- a/ui/tests/cutPaper.test.ts +++ b/ui/tests/cutPaper.test.ts @@ -14,7 +14,7 @@ import { createTijeralStoryProject, TIJERAL_STORY_ID } from '../src/features/cut import { normalizeStoryProject } from '../src/features/stories/model.ts' import { assertCutPaperKitHasNoPrivateGlb, cutPaperKitManifest, cutPaperPuppetLayers } from '../src/features/cutPaper/puppet.ts' import { parseSceneFile, serializeSceneFile } from '../src/lib/sceneFile.ts' -import { normalizeSceneKeyframes, withSceneKeyframes } from '../src/lib/sceneTimeline.ts' +import { evaluateSceneLayer, normalizeSceneKeyframes, withSceneKeyframes } from '../src/lib/sceneTimeline.ts' test('Tijeral kit ids are stable, original and not a private GLB body', () => { assert.equal(CUT_PAPER_KIT_ID, 'tijeral-cut-paper') @@ -94,6 +94,15 @@ test('Story Lab chapter roundtrips and each beat links to Video 2D', () => { assert.equal(talkEn.audioTracks?.[0]?.filename, 'vo-nilo-nilo-1-en.wav') assert.ok(talkEn.dialogueBeats?.some(beat => /fountain|frozen/i.test(beat.text))) assert.equal(CUT_PAPER_PILOT_SCRIPT_EN.length, CUT_PAPER_PILOT_SCRIPT.length) + const niloWide = talk.layers.find(layer => layer.id === 'puppet-nilo-mouth-wide') + const niloClosed = talk.layers.find(layer => layer.id === 'puppet-nilo-mouth-closed') + const bertaWide = talk.layers.find(layer => layer.id === 'puppet-berta-mouth-wide') + const kitoWide = sticker.layers.find(layer => layer.id === 'puppet-kito-mouth-wide') + assert.equal(evaluateSceneLayer(niloClosed, 1).opacity, 1) + assert.equal(evaluateSceneLayer(niloWide, 1).opacity, 0) + assert.equal(evaluateSceneLayer(niloWide, 6.05).opacity, 1) + assert.equal(evaluateSceneLayer(bertaWide, 1).opacity, 0) + assert.equal(evaluateSceneLayer(kitoWide, 1).opacity, 0) }) test('Tijeral character kits are 2D cutouts with Qwen voices and optional bodies', () => { diff --git a/ui/tests/cutoutDialogue.test.mjs b/ui/tests/cutoutDialogue.test.mjs index 9eee8f00..faf36a83 100644 --- a/ui/tests/cutoutDialogue.test.mjs +++ b/ui/tests/cutoutDialogue.test.mjs @@ -40,10 +40,37 @@ test('aligned words stay open during speech and close in the gaps', () => { { text: 'sticker', start: 10.52, end: 10.96 }, ], 30) const at = time => plan.visemes.find(beat => time >= beat.start && time < beat.end)?.state + assert.equal(at(1), 'closed') assert.equal(at(10.15), 'wide') assert.equal(at(10.4), 'round') assert.equal(at(10.7), 'wide') assert.equal(plan.visemes.at(-1).state, 'closed') + const starts = plan.visemes.map(beat => beat.start) + assert.equal(new Set(starts).size, starts.length) +}) + +test('aligned rest closed is a distinct keyframe so playback does not hold the first word open', () => { + const closed = { + id: 'mouth-closed', name: 'Closed', type: 'overlay', + transform: { x: 50, y: 48, scale: .12, opacity: 0, rotation: 0 }, + animation: { start: { x: 50, y: 48, scale: .12, opacity: 0 }, end: { x: 50, y: 48, scale: .12, opacity: 0 }, duration: 20, curve: 'hold' }, + faceBinding: { poseLayerId: 'pose', role: 'mouth', state: 'closed' }, + } + const wide = { + ...closed, id: 'mouth-wide', name: 'Wide', + faceBinding: { poseLayerId: 'pose', role: 'mouth', state: 'wide' }, + } + const beats = [ + { id: 'w0', text: 'La', start: 6, end: 6.14, mouthLayerIds: [closed.id, wide.id], confidence: 'aligned-audio' }, + { id: 'w1', text: 'fuente', start: 6.14, end: 6.56, mouthLayerIds: [closed.id, wide.id], confidence: 'aligned-audio' }, + ] + const rebuilt = rebuildCutoutDialogueLayers([closed, wide], beats, 30, 20) + const closedLayer = rebuilt.find(item => item.id === closed.id) + const wideLayer = rebuilt.find(item => item.id === wide.id) + assert.equal(evaluateSceneLayer(closedLayer, 1).opacity, 1) + assert.equal(evaluateSceneLayer(wideLayer, 1).opacity, 0) + assert.equal(evaluateSceneLayer(wideLayer, 6.05).opacity, 1) + assert.equal(evaluateSceneLayer(closedLayer, 6.05).opacity, 0) }) test('a long mixed-vowel line retains every available mouth family', () => {