Skip to content

Commit 0aae937

Browse files
authored
Merge pull request #265 from IAnMove/codex/3d-speech-productions
feat(video3d): lip-sync local, expresiones y planos de diálogo
2 parents f2ef220 + d77f89e commit 0aae937

89 files changed

Lines changed: 4404 additions & 52 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/workflows/ci.yml

Lines changed: 42 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -164,11 +164,15 @@ jobs:
164164
- name: Install Playwright Chromium
165165
working-directory: ui
166166
run: npx playwright install --with-deps chromium
167+
- name: Install Chrome for native H.264 and AAC speech export checks
168+
working-directory: ui
169+
run: npx playwright install chrome
167170
- name: UI E2E
168171
working-directory: ui
169172
run: npm run test:e2e
170173
- name: Upload Playwright artifacts
171-
if: failure()
174+
# Passing speech tests also attach screenshots, native JSON and MP4 evidence.
175+
if: always()
172176
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
173177
with:
174178
name: ui-e2e-artifacts
@@ -177,6 +181,40 @@ jobs:
177181
ui/test-results
178182
if-no-files-found: ignore
179183

184+
ui-speech-windows:
185+
name: Speech E2E Windows (real H.264 + AAC)
186+
runs-on: windows-2025
187+
timeout-minutes: 15
188+
env:
189+
# This job may not pass via the unsupported-codec assertion.
190+
HOCUSPOCUS_REQUIRE_SPEECH_AAC: "1"
191+
steps:
192+
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
193+
- uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0
194+
with:
195+
node-version: "24.18.0"
196+
check-latest: false
197+
cache: npm
198+
cache-dependency-path: ui/package-lock.json
199+
- name: Install UI deps
200+
working-directory: ui
201+
run: npm ci
202+
- name: Install Chromium for screen regression and Edge for native AAC
203+
working-directory: ui
204+
run: npx playwright install chromium msedge
205+
- name: Speech E2E with real export required
206+
working-directory: ui
207+
run: npm run test:e2e -- scene3d-speech.spec.ts scene3d-media-screen.spec.ts --workers=1
208+
- name: Upload speech evidence
209+
if: always()
210+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
211+
with:
212+
name: ui-speech-windows-artifacts
213+
path: |
214+
ui/playwright-report
215+
ui/test-results
216+
if-no-files-found: ignore
217+
180218
code-health-comment:
181219
name: Code-health PR comment
182220
if: ${{ github.event_name == 'pull_request' && always() && github.event.pull_request.head.repo.full_name == github.repository }}
@@ -207,7 +245,7 @@ jobs:
207245
ci-required:
208246
name: CI required
209247
if: always()
210-
needs: [guard, ui-check, ui-e2e]
248+
needs: [guard, ui-check, ui-e2e, ui-speech-windows]
211249
runs-on: ubuntu-24.04
212250
timeout-minutes: 5
213251
steps:
@@ -217,4 +255,5 @@ jobs:
217255
python3 scripts/ci_required.py \
218256
"Clean-repo guard + Python checks=${{ needs.guard.result }}" \
219257
"UI tests + lint + type-check + build=${{ needs.ui-check.result }}" \
220-
"UI E2E boot (Chromium + simulated API)=${{ needs.ui-e2e.result }}"
258+
"UI E2E boot (Chromium + simulated API)=${{ needs.ui-e2e.result }}" \
259+
"Speech E2E Windows (real H.264 + AAC)=${{ needs.ui-speech-windows.result }}"

app/routers/character_kit_face.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77

88
from fastapi import APIRouter, HTTPException
99
from pydantic import BaseModel, Field
10+
from routers.scene3d_speech import create_scene3d_speech_router
11+
from routers.scene3d_profiles import create_scene3d_profiles_router
1012

1113
from services.character_kit_face_cleanup import (
1214
CharacterKitFaceCleanupError,
@@ -26,6 +28,8 @@ def create_character_kit_face_router(
2628
uploads_root: Callable[[], str],
2729
) -> APIRouter:
2830
router = APIRouter(prefix="/api/v1/character-kits", tags=["Character kits"])
31+
router.include_router(create_scene3d_speech_router())
32+
router.include_router(create_scene3d_profiles_router(workspace_dir))
2933

3034
@router.post("/face-rig/cleanup")
3135
def cleanup_face_rig_overlay(payload: FaceRigCleanupRequest):

app/routers/scene3d_profiles.py

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
"""Workspace-scoped, content-addressed face calibration. No audio or model bytes."""
2+
from __future__ import annotations
3+
import json
4+
import os
5+
import threading
6+
import uuid
7+
from pathlib import Path
8+
from fastapi import APIRouter, HTTPException
9+
from pydantic import BaseModel, ConfigDict, Field, field_validator
10+
from services.character_speech_definition import face_settings
11+
12+
_lock = threading.Lock()
13+
14+
class ProfileWrite(BaseModel):
15+
model_config = ConfigDict(extra="forbid")
16+
workspace: str = Field(min_length=1, max_length=120, pattern=r"^[A-Za-z0-9_. -]+$")
17+
revision: int = Field(ge=0)
18+
settings: dict
19+
20+
@field_validator("workspace")
21+
@classmethod
22+
def workspace_name(cls, value):
23+
if value in {".", ".."}:
24+
raise ValueError("Invalid workspace.")
25+
return value
26+
27+
@field_validator("settings")
28+
@classmethod
29+
def face_only(cls, value):
30+
return face_settings(value)
31+
32+
def create_scene3d_profiles_router(workspace_dir):
33+
router = APIRouter()
34+
35+
def target(workspace: str, digest: str):
36+
import re
37+
if not re.fullmatch(r"[a-f0-9]{64}", digest) or not re.fullmatch(r"[A-Za-z0-9_. -]{1,120}", workspace) or workspace in {".", ".."}:
38+
raise HTTPException(400, "Invalid profile scope.")
39+
return Path(workspace_dir(workspace)).resolve() / ".speech3d-profiles" / (digest + ".json")
40+
41+
def read(path):
42+
return json.loads(path.read_text(encoding="utf-8")) if path.is_file() else None
43+
44+
@router.get("/speech/profiles/{digest}")
45+
def get_profile(digest: str, workspace: str):
46+
with _lock:
47+
profile = read(target(workspace, digest))
48+
if profile is None:
49+
raise HTTPException(404, "No saved calibration for this model.")
50+
return profile
51+
52+
@router.put("/speech/profiles/{digest}")
53+
def put_profile(digest: str, payload: ProfileWrite):
54+
path = target(payload.workspace, digest)
55+
with _lock:
56+
current = read(path)
57+
revision = current["revision"] if current else 0
58+
if revision != payload.revision:
59+
raise HTTPException(409, "Calibration changed elsewhere; reload before saving.")
60+
result = {"version": 1, "digest": digest, "revision": revision + 1, "settings": payload.settings}
61+
path.parent.mkdir(parents=True, exist_ok=True)
62+
# Preserve earlier revisions; never delete a user calibration.
63+
if current:
64+
history = path.with_name(digest + ".v" + str(revision) + ".json")
65+
if not history.exists():
66+
history.write_text(json.dumps(current, allow_nan=False), encoding="utf-8")
67+
temp = path.with_name(digest + "." + uuid.uuid4().hex + ".tmp")
68+
temp.write_text(json.dumps(result, allow_nan=False), encoding="utf-8")
69+
os.replace(temp, path)
70+
return result
71+
return router

app/routers/scene3d_speech.py

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
"""Small router mounted by the existing character-kit boundary, without loading AI runtimes."""
2+
from fastapi import APIRouter, HTTPException, Request
3+
from pydantic import BaseModel, Field
4+
from starlette.concurrency import run_in_threadpool
5+
6+
from services.scene3d_speech import MAX_BYTES, SpeechAnalysisError, SpeechAnalysisUnavailable, analyze_voice
7+
8+
9+
class SpeechMouthCue(BaseModel):
10+
start: float = Field(ge=0)
11+
end: float = Field(gt=0)
12+
value: str = Field(pattern="^[ABCDEFGHX]$")
13+
14+
15+
class SpeechAnalysisResponse(BaseModel):
16+
mouthCues: list[SpeechMouthCue]
17+
recognizer: str = "phonetic"
18+
duration: float
19+
20+
21+
def create_scene3d_speech_router() -> APIRouter:
22+
router = APIRouter()
23+
24+
@router.post("/speech/analyze", response_model=SpeechAnalysisResponse)
25+
async def analyze(request: Request):
26+
if request.headers.get("content-type", "").split(";")[0] != "audio/wav":
27+
raise HTTPException(415, "Expected audio/wav.")
28+
data = bytearray()
29+
async for chunk in request.stream():
30+
if len(data) + len(chunk) > MAX_BYTES:
31+
raise HTTPException(413, "Voice clip exceeds the 90-second limit.")
32+
data.extend(chunk)
33+
try:
34+
return await run_in_threadpool(analyze_voice, bytes(data))
35+
except SpeechAnalysisError as exc:
36+
raise HTTPException(400, str(exc)) from exc
37+
except SpeechAnalysisUnavailable as exc:
38+
raise HTTPException(503, str(exc)) from exc
39+
40+
return router

app/services/character_kit_library.py

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
from typing import Any
1616

1717
from .character_face_patch import normalize_character_face_patch
18+
from .character_speech_definition import normalize_speech3d, normalize_character_voice
1819

1920

2021
CHARACTER_KIT_LIBRARY_FILENAME = ".character-kit-library-v1.json"
@@ -164,6 +165,12 @@ def normalize_character_kit(value: Any, fallback_id: str = "") -> dict[str, Any]
164165
}
165166
if len(result["provenance"]) > 500 or any(not isinstance(item, dict) for item in result["provenance"]):
166167
raise ValueError("Character Kit provenance must contain at most 500 objects")
168+
if value.get("speech3d") is not None:
169+
result["speech3d"] = normalize_speech3d(value["speech3d"])
170+
if value.get("voice") is not None:
171+
result["voice"] = normalize_character_voice(value["voice"])
172+
if value.get("lookNotes"):
173+
result["lookNotes"] = _text(value["lookNotes"], "Character look notes", 4000)
167174
for key in ("identityReference", "base"):
168175
if value.get(key) is not None:
169176
result[key] = _asset(value[key], f"Character Kit {key}")
@@ -224,6 +231,12 @@ def write_character_kit_library(workspace_dir: str, value: Any, *, base_revision
224231
os.makedirs(workspace_dir, exist_ok=True)
225232
path = character_kit_library_path(workspace_dir)
226233
temporary = f"{path}.{uuid.uuid4().hex}.tmp"
234+
# Keep the previous authored revision; never silently discard calibration.
235+
if current["revision"] > 0:
236+
history = f"{path}.v{current['revision']}.json"
237+
if not os.path.exists(history):
238+
with open(history, "x", encoding="utf-8") as handle:
239+
json.dump(current, handle, ensure_ascii=False, allow_nan=False)
227240
try:
228241
with open(temporary, "w", encoding="utf-8") as handle:
229242
handle.write(encoded)
Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
"""Public reusable 3D/voice metadata. No scene audio, credentials, or AI imports."""
2+
import json
3+
import math
4+
import re
5+
from urllib.parse import urlsplit, parse_qsl
6+
7+
8+
def _asset_fields(value):
9+
if not isinstance(value, dict):
10+
raise ValueError("A persistent asset reference is required.")
11+
if set(value) - {"workspaceId", "filename", "url", "assetId"}:
12+
raise ValueError("Invalid asset fields.")
13+
for key in ("workspaceId", "filename", "url"):
14+
if not isinstance(value.get(key), str) or not value[key] or len(value[key]) > 1200:
15+
raise ValueError("Invalid asset reference.")
16+
if "assetId" in value and (not isinstance(value["assetId"], str) or len(value["assetId"]) > 240):
17+
raise ValueError("Invalid asset identity.")
18+
19+
20+
def _persistent_url(url):
21+
parsed = urlsplit(url)
22+
if not (url.startswith("/") and not url.startswith("//") or parsed.scheme in {"http", "https"} and parsed.netloc):
23+
raise ValueError("Use a persistent HTTP asset.")
24+
if parsed.username or parsed.password or any(ord(c) <= 32 or c == "\\" for c in url):
25+
raise ValueError("Credentials are not asset references.")
26+
if any(re.search(r"token|key|secret|signature", key, re.I) for key, _ in parse_qsl(parsed.query)):
27+
raise ValueError("Import the asset instead of saving credential-bearing URLs.")
28+
29+
30+
def source_ref(value):
31+
_asset_fields(value)
32+
_persistent_url(value["url"])
33+
return dict(value)
34+
35+
36+
def _vector(raw, length, low, high):
37+
return isinstance(raw, list) and len(raw) == length and all(
38+
type(n) in (int, float) and math.isfinite(n) and low <= n <= high for n in raw)
39+
40+
41+
def _eye_placement(eyes):
42+
return (isinstance(eyes, dict) and set(eyes) == {"left", "right", "size", "skinLeft", "skinRight"}
43+
and all(_vector(eyes.get(k), 3, -10000, 10000) for k in ("left", "right"))
44+
and _vector(eyes.get("size"), 2, .00001, 10000)
45+
and all(_vector(eyes.get(k), 3, 0, 1) for k in ("skinLeft", "skinRight")))
46+
47+
48+
def _face_placement(face):
49+
if (set(face) != {"meshIndex", "center", "size", "skin", "eyes"}
50+
or type(face["meshIndex"]) is not int or not 0 <= face["meshIndex"] <= 1023
51+
or not _eye_placement(face.get("eyes"))
52+
or not _vector(face.get("center"), 3, -10000, 10000)
53+
or not _vector(face.get("size"), 2, .00001, 10000)
54+
or not _vector(face.get("skin"), 3, 0, 1)):
55+
raise ValueError("Invalid face placement.")
56+
57+
58+
def _face_style(value):
59+
if "atlas" in value:
60+
source_ref(value["atlas"])
61+
for key in ("clean", "blink", "eyes"):
62+
if key in value and type(value[key]) is not bool:
63+
raise ValueError("Invalid face switch.")
64+
if "strength" in value and (type(value["strength"]) not in (int, float) or not 0 <= value["strength"] <= 1.5):
65+
raise ValueError("Invalid face strength.")
66+
for key, choices in (("style", {"soft", "toon", "pixel"}), ("expression", {"neutral", "happy", "angry", "worried", "surprised", "sleepy"})):
67+
if key in value and (not isinstance(value[key], str) or value[key] not in choices):
68+
raise ValueError("Invalid face style.")
69+
if "lip" in value and not re.fullmatch(r"#[0-9a-fA-F]{6}", str(value["lip"])):
70+
raise ValueError("Invalid lip color.")
71+
72+
73+
def face_settings(value):
74+
allowed = {"face", "atlas", "strength", "clean", "style", "lip", "expression", "blink", "eyes"}
75+
if not isinstance(value, dict) or not isinstance(value.get("face"), dict) or not set(value).issubset(allowed):
76+
raise ValueError("Only face settings may be stored.")
77+
_face_placement(value["face"])
78+
_face_style(value)
79+
if len(json.dumps(value, allow_nan=False)) > 24000:
80+
raise ValueError("Face profile too large.")
81+
return value
82+
83+
84+
def normalize_speech3d(value):
85+
if not isinstance(value, dict) or set(value) - {"model", "digest", "settings"}:
86+
raise ValueError("Invalid 3D character definition.")
87+
model = source_ref(value.get("model"))
88+
if not model["filename"].lower().endswith(".glb") or not re.fullmatch("[a-f0-9]{64}", str(value.get("digest", ""))):
89+
raise ValueError("Save a GLB with its content digest.")
90+
result = {"model": model, "digest": value["digest"]}
91+
if value.get("settings") is not None:
92+
result["settings"] = face_settings(value["settings"])
93+
return result
94+
95+
96+
def normalize_character_voice(value):
97+
if not isinstance(value, dict) or set(value) - {"provider", "model", "voiceId", "instructions"}:
98+
raise ValueError("Store voice preferences only, never credentials.")
99+
if value.get("provider") != "local" or value.get("model") != "qwen3_tts_customvoice":
100+
raise ValueError("Choose a supported local character voice.")
101+
if value.get("voiceId") not in {"vivian", "serena", "uncle_fu", "dylan", "eric", "ryan", "aiden", "ono_anna", "sohee"}:
102+
raise ValueError("Invalid voice preset.")
103+
if "instructions" in value and (not isinstance(value["instructions"], str) or len(value["instructions"]) > 1000):
104+
raise ValueError("Voice instructions are too long.")
105+
return dict(value)

0 commit comments

Comments
 (0)