diff --git a/app/_launch_runtime.py b/app/_launch_runtime.py index 81254cee0..c60275312 100644 --- a/app/_launch_runtime.py +++ b/app/_launch_runtime.py @@ -10672,6 +10672,13 @@ def _run_generation_with_preparation(job_id: str) -> bool: job = _jobs.get(job_id) if not isinstance(job, dict): return False + try: + native_worker = _image_generation_commands.native_worker(job) + except HTTPException as error: + finish_job(job, "failed", error=str(error.detail), message="Command recovery could not be verified") + return False + if native_worker is not None: + return bool(native_worker(job_id)) params = job.get("params") if isinstance(job.get("params"), dict) else {} pending = params.get("_h3_window_plan_pending") if not isinstance(pending, dict): @@ -10870,7 +10877,9 @@ async def generate(request: Request): prepare_generation_inputs(body, _generation_model_def, requested_workspace, uploads_dir=os.path.join(os.getcwd(), "uploads"), workspace_dir=_workspace_dir(requested_workspace), - prepared_images=getattr(request, "prepared_studio_images", False) is True) + prepared_images=getattr(request, "prepared_studio_images", False) is True, + prepared_speech=(getattr(request, "prepared_studio_speech", False) is True + or getattr(request, "prepared_studio_audio", False) is True)) except ValueError as error: raise HTTPException(status_code=400, detail=str(error)) from error try: @@ -22438,6 +22447,13 @@ def _run_sfx_generation(job: dict, raw_params: dict, start_time: float): ): return False + from services.studio_sfx_execution import prepared_sfx_execution + from services.studio_sfx_commands import check_sfx_models + typed_sfx = prepared_sfx_execution( + job, raw_params, registry=_task_registry(job["workspace"]), + check_models=lambda variant: check_sfx_models(globals(), variant), + ) + out_dir = job.get("out_dir") or wgp.save_path os.makedirs(out_dir, exist_ok=True) wgp.save_path = out_dir @@ -22459,11 +22475,17 @@ def _run_sfx_generation(job: dict, raw_params: dict, start_time: float): video_path = candidate if video_path and not os.path.isfile(video_path): + if typed_sfx: + raise ValueError("The admitted SFX video is no longer available") print(f"[SFX] Warning: video_guide not found: {video_path}, falling back to text-only") video_path = None # If video provided, derive duration from it - if video_path: + if typed_sfx: + # The admission records the inspected guide duration and exact + # source identity. Do not rederive it or silently cap the request. + pass + elif video_path: try: import decord vr = decord.VideoReader(video_path) @@ -22494,12 +22516,14 @@ def _run_sfx_generation(job: dict, raw_params: dict, start_time: float): ) return False - # Download model files if needed - if not update_job( - job, message="Downloading MMAudio models...", phase="Downloading models", - ): - return False - wgp.download_mmaudio(variant_override=variant) + # Typed commands require installed dependencies, checked again above. + # Keep legacy provisioning until its callers have been migrated. + if not typed_sfx: + if not update_job( + job, message="Downloading MMAudio models...", phase="Downloading models", + ): + return False + wgp.download_mmaudio(variant_override=variant) if is_cancel_requested(job): return False @@ -22552,10 +22576,10 @@ def _run_sfx_generation(job: dict, raw_params: dict, start_time: float): elapsed = time.time() - start_time for fname in new_files: ext = os.path.splitext(fname)[1].lower() - if ext not in {".wav", ".mp3", ".flac"}: + if ext not in {".wav", ".mp3", ".flac", ".mp4"}: continue sidecar = { - "params": { + "params": copy.deepcopy(raw_params) if typed_sfx else { "prompt": prompt, "MMAudio_prompt": prompt, "MMAudio_neg_prompt": neg_prompt, @@ -23435,6 +23459,17 @@ async def tools_upscale(request: Request): "workspace": workspace, "out_dir": output_dir, "provenance": provenance, } + admit_command = getattr(request, "admit_generation_command", None) + if callable(admit_command): + # Only the typed in-process command adapter can transfer admission. + # Preserve the existing tool's resolved inputs and native worker; + # canonical task/receipt persistence now owns its queue lifecycle. + job["params"].pop("_non_durable_tool", None) + provenance["capability"] = "tools.upscale" + command_collection = (body.get("provenance") or {}).get("workspace_id") + if command_collection is not None: + provenance["workspace_id"] = command_collection + return admit_command(job["params"], workspace, provenance) _register_manual_generation_job(job) worker = _run_generation if execution_mode.policy().simulated else _run_tool_upscale threading.Thread(target=worker, args=(job_id,), daemon=False).start() @@ -36223,7 +36258,7 @@ def _generation_task_fields(job: dict) -> dict: }.get(mode, "Generation job") if str(provenance.get("capability") or "") == "remove_background": task_title = "Tools · Remove background" - elif str(provenance.get("capability") or "") == "upscale": + elif str(provenance.get("capability") or "") in {"upscale", "tools.upscale"}: task_title = "Tools · Upscale" elif str(provenance.get("capability") or "") == "revoice": task_title = "Tools · Revoice" @@ -36863,7 +36898,8 @@ def _classic_redirect(): "generate": generate, "recast": recast_endpoint, "upscale": tools_upscale, **wangp_agent_handlers(api), **image_command_handlers(_image_generation_commands)}, journal_path=os.path.join(os.path.dirname(__file__), "settings", "wangp-mcp-requests.sqlite3"), - command_operations=[*workspace_command_catalog()["operations"], *image_command_catalog()], + command_operations=[*workspace_command_catalog()["operations"], *image_command_catalog( + adapter.catalog for adapter in _image_generation_commands.operations.values())], )) # ============================================================================ diff --git a/app/routers/image_generation_commands.py b/app/routers/image_generation_commands.py index 9b32269f0..99409b15b 100644 --- a/app/routers/image_generation_commands.py +++ b/app/routers/image_generation_commands.py @@ -1,5 +1,6 @@ """HTTP and MCP projections of the executable image command contract.""" from __future__ import annotations +from typing import Literal from fastapi import APIRouter, Request from pydantic import BaseModel, ConfigDict, Field, StrictStr, ValidationError, field_validator @@ -11,6 +12,7 @@ class ReferenceResolutionInput(BaseModel): model_config = ConfigDict(extra="forbid", strict=True) references: list[StrictStr] = Field(min_length=1, max_length=64) + media_kind: Literal["image", "audio", "video"] = "image" class UISubmissionContext(BaseModel): @@ -37,7 +39,7 @@ def _ui_context(request): raise command_error(422, "invalid_ui_context", "Use only exact workflowId and runId attribution") from error -def image_command_catalog(): +def image_command_catalog(additional_operations=()): spec = image_generation_schema() studio = studio_image_schema() studio_input = dict(studio["input"]) @@ -60,18 +62,20 @@ def image_command_catalog(): "description": "Admit an image job with an installed model and explicit output workspace. Version 1 is a single text-to-image request; version 2 accepts the complete typed Studio image parameters, canonical references, LoRAs and image processors. Preserve literal prompts and reuse intent_id only for retries. The receipt proves admission; inspect its task for completion.", "inputSchema": envelope}, {"name": "generation.receipt", "version": 1, "domain": "studio", "mutation": False, - "description": "Read an immutable image admission and its current canonical task in the exact original output workspace.", + "description": "Read an immutable generation admission and its current canonical task in the exact original output workspace.", "inputSchema": {"type": "object", "additionalProperties": False, "properties": {"version": {"type": "integer", "const": 1}, "operation": {"const": "generation.receipt"}, "input": receipt_input}, - "required": ["version", "operation", "input"]}}] + "required": ["version", "operation", "input"]}}, *additional_operations] def image_command_handlers(service): - async def submit(arguments): - if not isinstance(arguments, dict) or set(arguments) != {"version", "intent_id", "input"}: - raise command_error(422, "invalid_command", "Use version, intent_id and input for the image tool") - return await service.submit({**arguments, "operation": "generation.image"}, trusted_tool="external_agent") + def submission_handler(operation): + async def submit(arguments): + if not isinstance(arguments, dict) or set(arguments) != {"version", "intent_id", "input"}: + raise command_error(422, "invalid_command", "Use version, intent_id and input for the generation tool") + return await service.submit({**arguments, "operation": operation}, trusted_tool="external_agent") + return submit def receipt(arguments): if (not isinstance(arguments, dict) or set(arguments) != {"version", "input"} @@ -81,7 +85,8 @@ def receipt(arguments): raise command_error(422, "invalid_command", "Use version 1 with workspace and intent_id") return service.receipt(**arguments["input"]) - return {"generation.image": submit, "generation.receipt": receipt} + operations = {"generation.image", *getattr(service, "operations", {})} + return {**{operation: submission_handler(operation) for operation in operations}, "generation.receipt": receipt} def create_image_generation_commands_router(service): @@ -89,7 +94,8 @@ def create_image_generation_commands_router(service): @router.get("/api/v1/generation/commands") def catalog(): - return {"version": 2, "operations": image_command_catalog()} + return {"version": 2, "operations": image_command_catalog( + adapter.catalog for adapter in getattr(service, "operations", {}).values())} @router.post("/api/v1/generation/commands") async def submit(request: Request): @@ -118,7 +124,8 @@ def references(body: ReferenceResolutionInput): if any(not 1 <= len(value) <= 8192 for value in body.references): raise command_error(422, "invalid_reference", "An exact bounded media reference is required") try: - return {"references": [resolve(value) for value in body.references]} + return {"references": [resolve(value) if body.media_kind == "image" + else resolve(value, media_kind=body.media_kind) for value in body.references]} except (ValueError, OSError) as error: raise command_error(422, "invalid_reference", str(error)) from error diff --git a/app/routers/studio_music_commands.py b/app/routers/studio_music_commands.py new file mode 100644 index 000000000..8aba3ebac --- /dev/null +++ b/app/routers/studio_music_commands.py @@ -0,0 +1,40 @@ +"""Serializable catalog for the local Studio music operation.""" + +from services.studio_music_spec import studio_music_schema + + +def music_command_catalog(): + """Return the exact closed ``generation.music`` discovery entry.""" + schema = studio_music_schema() + input_schema = dict(schema["input"]) + definitions = input_schema.pop("$defs", {}) + return { + "name": "generation.music", + "version": 2, + "supportedVersions": [2], + "domain": "studio", + "mutation": True, + "description": ( + "Admit one local music generation with literal lyrics and caption " + "through the canonical generation queue. The selected ACE-Step or " + "MiniMax-Music3 model must already be installed; retries reuse the " + "same intent receipt." + ), + "musicModelTypes": list(schema["music_model_types"]), + "guideRevision": schema["guide_revision"], + "inputSchema": { + "type": "object", + "additionalProperties": False, + "$defs": definitions, + "properties": { + "version": {"type": "integer", "const": 2}, + "operation": {"const": "generation.music"}, + "intent_id": schema["intent_id"], + "input": input_schema, + }, + "required": ["version", "operation", "intent_id", "input"], + }, + } + + +__all__ = ["music_command_catalog"] diff --git a/app/routers/studio_sfx_commands.py b/app/routers/studio_sfx_commands.py new file mode 100644 index 000000000..4d081dfed --- /dev/null +++ b/app/routers/studio_sfx_commands.py @@ -0,0 +1,20 @@ +"""Executable SFX command schema shared by local HTTP and external MCP.""" +from services.studio_sfx_spec import studio_sfx_schema + + +def sfx_command_catalog(): + schema = studio_sfx_schema() + input_schema = dict(schema["input"]) + definitions = input_schema.pop("$defs", {}) + return { + "name": "generation.sfx", "version": 2, "supportedVersions": [2], + "domain": "studio", "mutation": True, + "description": "Generate sound effects with installed MMAudio files in an explicit output workspace. Preserve literal prompts. Text-only requests produce audio with a duration up to 20 seconds; video-guided requests use the inspected video duration and produce a video with new audio. Sources require canonical references. Reuse intent_id only to recover an existing admission; follow its task for completion.", + "inputSchema": { + "type": "object", "additionalProperties": False, "$defs": definitions, + "properties": {"version": {"type": "integer", "const": 2}, + "operation": {"const": "generation.sfx"}, + "intent_id": schema["intent_id"], "input": input_schema}, + "required": ["version", "operation", "intent_id", "input"], + }, + } diff --git a/app/routers/studio_speech_commands.py b/app/routers/studio_speech_commands.py new file mode 100644 index 000000000..22f08ca47 --- /dev/null +++ b/app/routers/studio_speech_commands.py @@ -0,0 +1,20 @@ +"""Serializable speech operation projected through the shared HTTP/MCP router.""" +from services.studio_speech_spec import studio_speech_schema + + +def speech_command_catalog(): + schema = studio_speech_schema() + input_schema = dict(schema["input"]) + definitions = input_schema.pop("$defs", {}) + return { + "name": "generation.speech", "version": 2, "supportedVersions": [2], + "domain": "studio", "mutation": True, + "description": "Admit speech using an installed speech model, literal text, voice settings and canonical audio references in an explicit output workspace. Preserve the original speaker text separately from the effective native prompt. Reuse intent_id only for retries; the receipt proves admission, and its task reports completion.", + "inputSchema": { + "type": "object", "additionalProperties": False, "$defs": definitions, + "properties": {"version": {"type": "integer", "const": 2}, + "operation": {"const": "generation.speech"}, + "intent_id": schema["intent_id"], "input": input_schema}, + "required": ["version", "operation", "intent_id", "input"], + }, + } diff --git a/app/routers/tools_upscale_commands.py b/app/routers/tools_upscale_commands.py new file mode 100644 index 000000000..fbcf69385 --- /dev/null +++ b/app/routers/tools_upscale_commands.py @@ -0,0 +1,51 @@ +"""Pure catalog projection for the typed Tools upscale operation.""" + +from __future__ import annotations + +from copy import deepcopy + +from services.tools_upscale_spec import tools_upscale_schema + + +def tools_upscale_command_catalog() -> dict: + """Return the exact operation entry consumed by HTTP/MCP discovery.""" + schema = tools_upscale_schema() + input_schema = dict(schema["input"]) + definitions = input_schema.pop("$defs", {}) + return { + "name": "tools.upscale", + "version": 2, + "supportedVersions": [2], + "domain": "tools", + "mutation": True, + "description": ( + "Upscale one exact image or video source in an explicit output " + "workspace with an installed local processor. Preserve the " + "source workspace, processor settings and intent_id; the shared " + "receipt proves admission and its canonical task reports completion." + ), + "inputSchema": { + "type": "object", + "additionalProperties": False, + "$defs": definitions, + "properties": { + "version": {"type": "integer", "const": 2}, + "operation": {"const": "tools.upscale"}, + "intent_id": deepcopy(schema["intent_id"]), + "input": input_schema, + }, + "required": ["version", "operation", "intent_id", "input"], + }, + } + + +# Short aliases used by catalog exporters in adjacent command slices. +tools_upscale_catalog = tools_upscale_command_catalog +upscale_command_catalog = tools_upscale_command_catalog + + +__all__ = [ + "tools_upscale_catalog", + "tools_upscale_command_catalog", + "upscale_command_catalog", +] diff --git a/app/services/image_generation_commands.py b/app/services/image_generation_commands.py index cadbbe0f2..c81669352 100644 --- a/app/services/image_generation_commands.py +++ b/app/services/image_generation_commands.py @@ -1,4 +1,4 @@ -"""Shared image admission using native preparation, tasks and generation FIFO. +"""Shared native admission using preparation, tasks and the generation FIFO. The receipt proves admission. TaskRegistry remains the progress authority and the native generation queue remains the sole execution/recovery mechanism. @@ -37,7 +37,8 @@ def validate_image_model(params, *, model_definition, model_downloaded, allow_re class ImageGenerationCommands: def __init__(self, *, registry, prepare, preflight, make_job, task_fields, - dispatch, persist_recovery, active_job_ids, prepare_studio=None, runtime_defaults=None): + dispatch, persist_recovery, active_job_ids, prepare_studio=None, runtime_defaults=None, + operations=None): self.registry = registry self.prepare = prepare self.preflight = preflight @@ -48,6 +49,9 @@ def __init__(self, *, registry, prepare, preflight, make_job, task_fields, self.active_job_ids = active_job_ids self.prepare_studio = prepare_studio self.runtime_defaults = runtime_defaults or (lambda: {}) + self.operations = dict(operations or {}) + if "generation.image" in self.operations: + raise ValueError("The existing image contract cannot be overridden") self.owner = uuid.uuid4().hex def _registry(self, workspace): @@ -59,7 +63,7 @@ def _registry(self, workspace): @staticmethod def _validate_replay(entry, frozen): - if (entry["operation"] != "generation.image" or entry["digest"] != frozen["fingerprint"] + if (entry["operation"] != frozen["original"]["operation"] or entry["digest"] != frozen["fingerprint"] or entry["fingerprint_version"] != frozen["fingerprint_version"]): raise TaskCommandConflict("intent_id was already used with different parameters or preconditions") @@ -95,13 +99,15 @@ def _admit(self, frozen, body, workspace, provenance): registry = self._registry(workspace) provenance = deepcopy(provenance) provenance["command"]["command_id"] = frozen["original"]["intent_id"] - native_params = {**deepcopy(self.runtime_defaults()), **deepcopy(body)} + adapter = self.operations.get(frozen["original"]["operation"]) + defaults = self.runtime_defaults() if adapter is None or adapter.use_generation_defaults else {} + native_params = {**deepcopy(defaults), **deepcopy(body)} job = self.make_job(native_params, workspace, reserve_generation=False, publish_task=False, provenance=provenance) effective = deepcopy(frozen["effective"]) effective["runtime"] = {"params": deepcopy(job["params"]), "workspace": workspace, "provenance": deepcopy(job["provenance"])} admitted = registry.admit_command_task( - intent_id=frozen["original"]["intent_id"], operation="generation.image", + intent_id=frozen["original"]["intent_id"], operation=frozen["original"]["operation"], digest=frozen["fingerprint"], original=frozen["original"], effective=effective, task_fields=self.task_fields(job), fingerprint_version=frozen["fingerprint_version"], ) @@ -116,7 +122,7 @@ def _provenance(frozen, trusted_tool, context): if context and context.get(source): command[target] = context[source] result = {"actor": "wizard" if trusted_tool == "wizard" else "user", - "capability": "generation.image", "command": command} + "capability": frozen["original"]["operation"], "command": command} collection = frozen["original"]["input"].get("workspace_collection_id") if collection is not None: result["workspace_id"] = collection @@ -131,7 +137,11 @@ async def submit(self, command, *, trusted_tool=None, submission_context=None): self._validate_replay(previous, frozen) self._dispatch_admitted(registry, previous) return {"receipt": previous["receipt"], "replayed": True} - if command["version"] == 2: + adapter = self.operations.get(command["operation"]) + if adapter is not None: + params, resources = adapter.prepare(params) + frozen["effective"]["resources"] = resources + elif command["version"] == 2: if self.prepare_studio is None: raise command_error(422, "unsupported_version", "Studio image commands are unavailable in this runtime") params, resources = self.prepare_studio(params) @@ -140,12 +150,15 @@ async def submit(self, command, *, trusted_tool=None, submission_context=None): self.preflight(params) request = JsonRequest({**deepcopy(params), "provenance": self._provenance( frozen, trusted_tool, submission_context)}, trusted_tool=trusted_tool) - request.prepared_studio_images = command["version"] == 2 + request.prepared_studio_images = command["operation"] == "generation.image" and command["version"] == 2 + request.prepared_studio_speech = command["operation"] == "generation.speech" + request.prepared_studio_audio = command["operation"] == "generation.music" # This callback is an in-process capability, never a JSON option. # The native facade performs its ordinary validation first and then # transfers admission to the same canonical task/worker adapter. request.admit_generation_command = lambda body, workspace, provenance: self._admit(frozen, body, workspace, provenance) - return await self.prepare(request) + prepare_request = adapter.prepare_request if adapter and adapter.prepare_request else self.prepare + return await prepare_request(request) except ImageGenerationSpecError as error: raise command_error(422, "invalid_command", str(error)) from error except TaskCommandConflict as error: @@ -153,8 +166,11 @@ async def submit(self, command, *, trusted_tool=None, submission_context=None): except (OSError, sqlite3.Error) as error: raise command_error(503, "storage_unavailable", "Command storage is unavailable; retry with the same intention") from error - @staticmethod - def _freeze(command): + def _freeze(self, command): + if isinstance(command, dict) and isinstance(command.get("operation"), str): + adapter = self.operations.get(command["operation"]) + if adapter is not None: + return adapter.freeze(command) if isinstance(command, dict) and type(command.get("version")) is int and command["version"] == 2: from services.studio_image_spec import freeze_studio_image_spec frozen = freeze_studio_image_spec(command) @@ -177,6 +193,25 @@ def receipt(self, workspace, intent_id): except (OSError, sqlite3.Error) as error: raise command_error(503, "storage_unavailable", "Command storage is unavailable") from error + def native_worker(self, job): + """Select a tool worker only for its real durable admission. + + Public generation JSON and legacy provenance can never select a tool + by themselves. Both normal dispatch and queue recovery check the + existing canonical receipt before entering the registered worker. + """ + provenance = job.get("provenance") + if not isinstance(provenance, dict) or not isinstance(provenance.get("capability"), str): + return None + operation = provenance["capability"] + adapter = self.operations.get(operation) + if adapter is None or adapter.worker is None: + return None + linked = self._recovery_task(job) + if not linked or linked[1] is None: + raise command_error(503, "recovery_mismatch", "Tool worker requires its canonical task") + return adapter.worker + def restore_recovery(self, workspaces): """Rebuild only the existing recovery projection; never start inference.""" try: @@ -196,6 +231,10 @@ def _restore_recovery(self, workspaces): continue raise for entry in registry.command_recovery_candidates(): + if entry["operation"] not in {"generation.image", *self.operations}: + # Another domain can share TaskRegistry without using + # this runtime's native generation recovery projection. + continue task = registry.get(entry["task_id"]) if not task or task["status"] != "interrupted" or task["backend_job_id"] in active: continue @@ -205,8 +244,7 @@ def _restore_recovery(self, workspaces): self.persist_recovery({"id": task["backend_job_id"], "status": "interrupted", "created_at": task["created_at"], **deepcopy(runtime)}) - @staticmethod - def _recovery_identity(record): + def _recovery_identity(self, record): if not isinstance(record, dict): return False provenance = record.get("provenance") @@ -217,7 +255,7 @@ def _recovery_identity(record): capability = provenance.get("capability") if capability is not None and not isinstance(capability, str): return False - if capability != "generation.image": + if capability not in {"generation.image", *self.operations}: return None command = provenance.get("command") if not isinstance(command, dict): @@ -230,8 +268,8 @@ def _recovery_identity(record): def _recovery_task(self, record): """Link one leftover to its admission, or withhold it. - Non-image leftovers return None so the native queue can recover them. - A linked image leftover returns ``(registry, task)``. An image row that + Unregistered operations return None for legacy native recovery. + A linked command returns ``(registry, task)``. A registered row that cannot be matched (missing admission, invalid queue metadata or job-id drift) returns False so this one row is skipped. Storage failures, including corrupt canonical admissions, remain errors: discard must @@ -243,7 +281,8 @@ def _recovery_task(self, record): try: registry = self._registry(record.get("workspace")) entry = registry.command_admission(intent_id) - if entry is None or entry["receipt"]["result"]["job_id"] != record.get("id"): + if (entry is None or entry["operation"] != record["provenance"]["capability"] + or entry["receipt"]["result"]["job_id"] != record.get("id")): return False return registry, registry.get(entry["task_id"]) except HTTPException as error: diff --git a/app/services/image_generation_runtime.py b/app/services/image_generation_runtime.py index a1e762221..7bfc27661 100644 --- a/app/services/image_generation_runtime.py +++ b/app/services/image_generation_runtime.py @@ -45,9 +45,13 @@ def preflight(params): validate_image_model(params, model_definition=runtime["wgp"].get_model_def, model_downloaded=runtime["_check_model_downloaded"]) - def resources(): + def resources(media_kind="image"): from services.studio_image_resources import StudioImageResources - return StudioImageResources( + from services.studio_speech_resources import StudioSpeechResources + from services.studio_sfx_resources import StudioSfxResources + resource_type = {"image": StudioImageResources, "audio": StudioSpeechResources, + "video": StudioSfxResources}[media_kind] + return resource_type( workspace_dir=runtime["_workspace_dir"], uploads_dir=lambda: os.path.join(os.getcwd(), "uploads"), list_workspaces=runtime["_list_workspaces"], lora_search_dirs=runtime["wgp"].get_lora_search_dirs, lora_compatible=runtime["_lora_is_compatible_with_model"], @@ -63,11 +67,49 @@ def prepare_studio(params): validate_processors=processors.validate_selection, processor_settings=processors.validated_settings, ) + def audio_operation(freeze_spec, prepare_audio, catalog): + from services.native_generation_operation import NativeGenerationOperation + + def freeze(command): + frozen = freeze_spec(command) + effective = frozen["effective"]["input"] + return frozen, {**deepcopy(effective["params"]), "workspace": effective["workspace"]} + + def prepare(params): + return prepare_audio(params, model_definition=runtime["wgp"].get_model_def, + model_downloaded=runtime["_check_model_downloaded"], + resources=resources("audio"), execution_policy=execution_policy) + + return NativeGenerationOperation(freeze=freeze, prepare=prepare, catalog=catalog()) + + from services.studio_speech_spec import freeze_studio_speech_spec + from services.studio_speech_preparation import prepare_studio_speech + from routers.studio_speech_commands import speech_command_catalog + from services.studio_music_spec import freeze_studio_music_spec + from services.studio_music_preparation import prepare_studio_music + from routers.studio_music_commands import music_command_catalog + + operations = { + "generation.speech": audio_operation(freeze_studio_speech_spec, prepare_studio_speech, speech_command_catalog), + "generation.music": audio_operation(freeze_studio_music_spec, prepare_studio_music, music_command_catalog), + } + + if callable(runtime.get("_run_generation")): + from services.studio_sfx_commands import create_sfx_operation + operations["generation.sfx"] = create_sfx_operation( + runtime, resources=lambda: resources("video"), execution_policy=execution_policy, + ) + + if callable(runtime.get("tools_upscale")) and callable(runtime.get("_run_tool_upscale")): + from services.tools_upscale_commands import create_tools_upscale_operation + operations["tools.upscale"] = create_tools_upscale_operation(runtime) + service = ImageGenerationCommands( registry=runtime["_task_registry"], prepare=runtime["generate"], preflight=preflight, make_job=runtime["_new_generation_job"], task_fields=runtime["_generation_task_fields"], dispatch=dispatch, persist_recovery=persist, active_job_ids=lambda: runtime["_jobs"].keys(), prepare_studio=prepare_studio, runtime_defaults=lambda: {"mode": "", **runtime["wgp"].primary_settings}, + operations=operations, ) - service.canonicalize_reference = lambda value: resources().canonicalize_legacy(value) + service.canonicalize_reference = lambda value, media_kind="image": resources(media_kind).canonicalize_legacy(value) return service diff --git a/app/services/native_generation_operation.py b/app/services/native_generation_operation.py new file mode 100644 index 000000000..17f78def2 --- /dev/null +++ b/app/services/native_generation_operation.py @@ -0,0 +1,17 @@ +"""Typed adapters for operations admitted into the existing generation queue. + +An adapter freezes caller input without I/O and validates/resolves resources +before native admission. It neither owns task storage nor starts workers. +""" +from dataclasses import dataclass +from typing import Callable + + +@dataclass(frozen=True) +class NativeGenerationOperation: + freeze: Callable[[dict], tuple[dict, dict]] + prepare: Callable[[dict], tuple[dict, list]] + catalog: dict + prepare_request: Callable | None = None + worker: Callable[[str], bool] | None = None + use_generation_defaults: bool = True diff --git a/app/services/studio_image_resources.py b/app/services/studio_image_resources.py index a1c1d0b6a..370582612 100644 --- a/app/services/studio_image_resources.py +++ b/app/services/studio_image_resources.py @@ -32,6 +32,8 @@ def file_identity(path): class StudioImageResources: + media_kind = "image" + def __init__(self, *, workspace_dir, uploads_dir, list_workspaces, lora_search_dirs, lora_compatible): self.workspace_dir = workspace_dir @@ -91,8 +93,8 @@ def _asset_url(self, identity): roots = [{"workspace_id": name, "path": self.workspace_dir(name)} for name in self._workspace_names()] roots.append({"workspace_id": "__uploads__", "path": self.uploads_dir()}) asset = find_asset(roots, identity) - if not asset or asset.get("kind") != "image": - raise ValueError("Choose an existing image asset ID") + if not asset or asset.get("kind") != self.media_kind: + raise ValueError(f"Choose an existing {self.media_kind} asset ID") locations = asset.get("locations") or [] if len(locations) != 1: raise ValueError("This asset has multiple locations; choose an exact source URL") diff --git a/app/services/studio_music_preparation.py b/app/services/studio_music_preparation.py new file mode 100644 index 000000000..04845d740 --- /dev/null +++ b/app/services/studio_music_preparation.py @@ -0,0 +1,384 @@ +"""Provider-free preflight for the closed Studio music command. + +This boundary checks the selected local native handler and model-declared +controls, then delegates canonical audio/LoRA inspection to the existing +``StudioSpeechResources`` implementation. It does not load weights, call a +provider, submit a task or create a second queue. +""" + +from __future__ import annotations + +from copy import deepcopy +import math +from typing import Any, Mapping + +from fastapi import HTTPException + +from services.image_generation_commands import command_error +from services.music_model_contract import ( + ACE_DEFAULT, + MUSIC3_LOCAL, + MusicModelError, + assert_enqueue_guard, + require_catalog_entry, +) +from services.lyrics_language import validate_lyrics_language +from services.studio_image_resources import validate_lora_multipliers +from services.studio_music_spec import STUDIO_MUSIC_MODEL_TYPES + + +_MODEL_DEFAULTS: dict[str, dict[str, Any]] = { + ACE_DEFAULT: { + "duration_seconds": 120.0, + "num_inference_steps": 8, + "guidance_scale": 1.0, + "guidance_phases": 1, + }, + MUSIC3_LOCAL: { + "duration_seconds": 120.0, + "num_inference_steps": 30, + "guidance_scale": 1.7, + "guidance_phases": 0, + }, +} +_AUDIO_REFERENCE_FIELDS = ("audio_guide", "audio_guide2", "audio_guide3", + "audio_guide4", "audio_guide5", "audio_guide6") + + +def _definition_for(model_definition, model_type: str) -> dict[str, Any]: + if callable(model_definition): + definition = model_definition(model_type) + elif isinstance(model_definition, Mapping): + definition = model_definition.get(model_type) + else: + definition = None + if not isinstance(definition, dict): + raise ValueError("Choose an installed local music model from the catalog") + return deepcopy(definition) + + +def _finite(value: Any, field: str, *, minimum: float | None = None, + maximum: float | None = None) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"input.params.{field} must be a finite number") + parsed = float(value) + if not math.isfinite(parsed): + raise ValueError(f"input.params.{field} must be a finite number") + if minimum is not None and parsed < minimum: + raise ValueError(f"input.params.{field} is below the model's declared minimum") + if maximum is not None and parsed > maximum: + raise ValueError(f"input.params.{field} exceeds the model's declared maximum") + return parsed + + +def _choice_values(choices: Any) -> list[Any]: + if not isinstance(choices, (list, tuple)): + return [] + values: list[Any] = [] + for item in choices: + if isinstance(item, Mapping): + item = item.get("value") + elif isinstance(item, (list, tuple)) and len(item) >= 2: + item = item[1] + values.append(item) + return values + + +def _validate_model(model_type: str, definition: dict[str, Any], model_downloaded) -> None: + if model_type not in STUDIO_MUSIC_MODEL_TYPES: + raise ValueError("Choose a registered local music model; remote models use another operation") + if definition.get("audio_only") is not True or definition.get("image_outputs"): + raise ValueError("The selected model is not an audio-only music model") + architecture = definition.get("architecture") + if model_type == MUSIC3_LOCAL and architecture not in (None, MUSIC3_LOCAL): + raise ValueError("The selected model definition does not match MiniMax-Music3") + if model_type == ACE_DEFAULT and architecture is not None and "ace_step" not in str(architecture): + raise ValueError("The selected model definition does not match ACE-Step") + if not callable(model_downloaded) or not model_downloaded(model_type): + raise command_error(409, "model_unavailable", "Required music model files are not installed; install them before submitting") + + +def _duration_bounds(model_type: str, definition: Mapping[str, Any]) -> tuple[float, float]: + entry = require_catalog_entry(model_type) + slider = definition.get("duration_slider") + # ``duration_min`` in music_model_contract is the Story policy (20s). + # Studio must use the selected native handler's declared slider instead: + # both local handlers currently accept 5s, and a future handler may have a + # different bound. Keep the catalog maximum as a safety ceiling while + # never raising the native minimum to satisfy Story's longer cue policy. + # The two registered local handlers both declare a 5s native minimum. If + # an older catalog projection omits the slider, retain that handler-backed + # minimum rather than reopening the Story policy or accepting unsupported + # sub-five-second requests. + lower = 5.0 + upper = float(entry["duration_max"]) + if isinstance(slider, Mapping): + if slider.get("min") is not None: + lower = float(slider["min"]) + if slider.get("max") is not None: + upper = min(upper, float(slider["max"])) + if lower > upper: + raise ValueError("The selected music model advertises invalid duration bounds") + return lower, upper + + +def _fill_duration(working: dict[str, Any], model_type: str, + definition: Mapping[str, Any]) -> None: + value = working.get("duration_seconds") + if value is None: + slider = definition.get("duration_slider") + value = slider.get("default") if isinstance(slider, Mapping) else None + value = _MODEL_DEFAULTS[model_type]["duration_seconds"] if value is None else value + working["duration_seconds"] = value + lower, upper = _duration_bounds(model_type, definition) + _finite(value, "duration_seconds", minimum=lower, maximum=upper) + + +def _fill_steps(working: dict[str, Any], model_type: str, + definition: Mapping[str, Any]) -> None: + value = working.get("num_inference_steps") + if value is None: + value = _MODEL_DEFAULTS[model_type]["num_inference_steps"] + working["num_inference_steps"] = value + if type(value) is not int or value < 1: + raise ValueError("input.params.num_inference_steps must be an integer from 1 upward") + lower = definition.get("inference_steps_min") + upper = definition.get("inference_steps_max") + if lower is not None and value < int(lower): + raise ValueError("input.params.num_inference_steps is below the model's declared minimum") + if upper is not None and value > int(upper): + raise ValueError("input.params.num_inference_steps exceeds the model's declared maximum") + if model_type == MUSIC3_LOCAL and value > 100: + raise ValueError("input.params.num_inference_steps exceeds MiniMax-Music3's maximum") + + +def _fill_guidance(working: dict[str, Any], model_type: str, + definition: Mapping[str, Any]) -> int: + if working.get("guidance_scale") is None: + working["guidance_scale"] = _MODEL_DEFAULTS[model_type]["guidance_scale"] + _finite(working["guidance_scale"], "guidance_scale", minimum=0, maximum=1000) + if definition.get("lock_guidance_scale") and model_type == MUSIC3_LOCAL: + expected = _MODEL_DEFAULTS[model_type]["guidance_scale"] + if float(working["guidance_scale"]) != expected: + raise ValueError("input.params.guidance_scale is locked by MiniMax-Music3") + phases = working.get("guidance_phases") + if phases is None: + phases = definition.get("guidance_max_phases", _MODEL_DEFAULTS[model_type]["guidance_phases"]) + working["guidance_phases"] = int(phases) + if type(phases) is not int or not 0 <= phases <= 16: + raise ValueError("input.params.guidance_phases must be an integer from 0 to 16") + maximum = definition.get("guidance_max_phases") + if maximum is not None and phases > int(maximum): + raise ValueError("input.params.guidance_phases exceeds the model's declared maximum") + return max(1, int(maximum if maximum is not None else phases or 1)) + + +def _validate_temperature(value: Any, definition: Mapping[str, Any]) -> None: + if value is None: + return + _finite(value, "temperature", minimum=0, maximum=2) + if definition.get("temperature") is False: + raise ValueError("input.params.temperature is locked for this music model") + + +def _validate_top_sampling(working: dict[str, Any], definition: Mapping[str, Any]) -> None: + top_p = working.get("top_p") + if top_p is not None: + _finite(top_p, "top_p", minimum=0, maximum=1) + if not definition.get("top_p_slider"): + raise ValueError("input.params.top_p is not supported by this music model") + top_k = working.get("top_k") + if top_k is not None: + if type(top_k) is not int or top_k < 0: + raise ValueError("input.params.top_k must be a non-negative integer") + if not definition.get("top_k_slider"): + raise ValueError("input.params.top_k is not supported by this music model") + + +def _validate_audio_sampling(working: dict[str, Any], definition: Mapping[str, Any]) -> None: + for field in ("audio_scale", "alt_guidance_scale"): + value = working.get(field) + if value is None: + continue + _finite(value, field, minimum=0, maximum=1000) + capability = "audio_scale_name" if field == "audio_scale" else "alt_guidance" + if not definition.get(capability): + raise ValueError(f"input.params.{field} is not supported by this music model") + + +def _validate_sampling(working: dict[str, Any], definition: Mapping[str, Any]) -> None: + solver = working.get("sample_solver") or "" + choices = _choice_values(definition.get("sample_solvers")) + if solver and (not choices or solver not in choices): + raise ValueError("input.params.sample_solver is not one of the model's declared choices") + if working.get("negative_prompt") not in (None, ""): + raise ValueError("input.params.negative_prompt is not supported by music models") + _validate_temperature(working.get("temperature"), definition) + _validate_top_sampling(working, definition) + _validate_audio_sampling(working, definition) + + +def _validate_model_mode(working: dict[str, Any], definition: Mapping[str, Any]) -> None: + value = working.get("model_mode") + if value is None: + modes = definition.get("model_modes") + default = modes.get("default") if isinstance(modes, Mapping) else None + if default is not None: + working["model_mode"] = default + return + modes = definition.get("model_modes") + choices = _choice_values(modes.get("choices") if isinstance(modes, Mapping) else None) + if not choices or value not in choices: + raise ValueError("input.params.model_mode is not one of the model's declared choices") + + +def _validate_music3_references(working: dict[str, Any], mode: str, + refs: Mapping[str, Any], active: Mapping[str, Any]) -> None: + if mode or active: + raise ValueError("MiniMax-Music3 does not support reference audio") + for field in _AUDIO_REFERENCE_FIELDS: + if refs[field] == "": + working[field] = None + + +def _validate_ace_reference_slots(mode: str, refs: Mapping[str, Any], + active: Mapping[str, Any], + definition: Mapping[str, Any]) -> None: + source = definition.get("audio_prompt_type_sources") + choices = _choice_values(source.get("selection") if isinstance(source, Mapping) else None) + if choices and mode not in choices: + raise ValueError("input.params.audio_prompt_type is not a declared ACE-Step choice") + extra = next((field for field in _AUDIO_REFERENCE_FIELDS[2:] + if refs[field] not in (None, "")), None) + if extra is not None: + raise ValueError(f"input.params.{extra} is not supported by ACE-Step music") + required = (("A", "audio_guide"), ("B", "audio_guide2")) + for marker, field in required: + if marker in mode and not refs[field]: + raise ValueError(f"input.params.audio_prompt_type requires {field}") + if marker not in mode and refs[field]: + raise ValueError(f"{field} requires its audio_prompt_type selector") + if not mode and active: + raise ValueError("audio references require an audio_prompt_type selector") + + +def _validate_audio_references(working: dict[str, Any], model_type: str, + definition: Mapping[str, Any]) -> None: + mode = working.get("audio_prompt_type") or "" + refs = {field: working.get(field) for field in _AUDIO_REFERENCE_FIELDS} + active = {field: value for field, value in refs.items() if value not in (None, "")} + if model_type == MUSIC3_LOCAL: + _validate_music3_references(working, mode, refs, active) + return + _validate_ace_reference_slots(mode, refs, active, definition) + + +def _setting_id(setting: Mapping[str, Any], index: int) -> str: + for key in ("id", "param", "name"): + value = setting.get(key) + if isinstance(value, str) and value.strip(): + return value.strip().lower().replace(" ", "_") + return f"custom_setting_{index + 1}" + + +def _validate_custom_value(key: str, value: Any, setting: Mapping[str, Any]) -> None: + setting_type = str(setting.get("type", "text")).lower() + if value == "" and setting_type in {"int", "float"}: + return + if setting_type == "int": + if type(value) is not int: + raise ValueError(f"input.params.custom_settings.{key} must be an integer") + numeric = float(value) + elif setting_type == "float": + numeric = _finite(value, f"custom_settings.{key}") + elif setting_type == "text": + if not isinstance(value, str): + raise ValueError(f"input.params.custom_settings.{key} must be text") + numeric = None + else: + raise ValueError(f"input.params.custom_settings.{key} has an unsupported model setting type") + if numeric is not None: + lower = setting.get("min", setting.get("minimum")) + upper = setting.get("max", setting.get("maximum")) + if lower is not None and numeric < float(lower): + raise ValueError(f"input.params.custom_settings.{key} is below its declared minimum") + if upper is not None and numeric > float(upper): + raise ValueError(f"input.params.custom_settings.{key} exceeds its declared maximum") + + +def _validate_custom_settings(working: dict[str, Any], definition: Mapping[str, Any]) -> None: + values = working.get("custom_settings") + if values in (None, {}): + return + definitions = definition.get("custom_settings") + if not isinstance(definitions, list): + raise ValueError("input.params.custom_settings is not supported by this music model") + known = {_setting_id(item, index): item for index, item in enumerate(definitions) + if isinstance(item, Mapping)} + unknown = sorted(set(values) - set(known)) + if unknown: + raise ValueError("input.params.custom_settings contains an unknown model setting") + for key, value in values.items(): + _validate_custom_value(key, value, known[key]) + + +def _validate_loras(working: dict[str, Any], definition: Mapping[str, Any], phases: int) -> None: + names = working.get("activated_loras") or [] + if (names or working.get("loras_multipliers")) and not definition.get("enabled_audio_lora"): + raise ValueError("selected LoRAs are not supported by this music model") + if names or working.get("loras_multipliers"): + validate_lora_multipliers(working, phases) + + +def _validate_lyrics_guard(working: Mapping[str, Any]) -> None: + language = working.get("lyrics_language") or "" + report = validate_lyrics_language( + str(working.get("prompt") or ""), + str(language), + instrumental=bool(working.get("_music_instrumental")), + ) + assert_enqueue_guard({ + "lyrics": working.get("prompt", ""), + "lyrics_language": language, + "instrumental": bool(working.get("_music_instrumental")), + "language_guard": report, + }) + + +def prepare_studio_music(params, *, model_definition, model_downloaded, resources, + execution_policy): + """Return detached native music parameters and inspected resources.""" + if not isinstance(params, dict): + raise command_error(422, "invalid_studio_music_input", "Music parameters must be an object") + working = deepcopy(params) + try: + workspace = working.get("workspace") + if not isinstance(workspace, str) or not workspace.strip(): + raise ValueError("input.workspace must be an explicit output workspace") + execution_policy(workspace) + model_type = working.get("model_type") + if not isinstance(model_type, str): + raise ValueError("input.params.model_type must be a string") + definition = _definition_for(model_definition, model_type) + _validate_model(model_type, definition, model_downloaded) + _fill_duration(working, model_type, definition) + _fill_steps(working, model_type, definition) + phases = _fill_guidance(working, model_type, definition) + _validate_sampling(working, definition) + _validate_model_mode(working, definition) + _validate_audio_references(working, model_type, definition) + _validate_custom_settings(working, definition) + _validate_lyrics_guard(working) + _validate_loras(working, definition, phases) + prepared, media = resources.prepare_media(working) + loras = resources.prepare_loras(working, definition) + if not isinstance(prepared, dict) or not isinstance(media, list) or not isinstance(loras, list): + raise ValueError("music resource preparation returned invalid native parameters") + return deepcopy(prepared), [*deepcopy(media), *deepcopy(loras)] + except HTTPException: + raise + except (MusicModelError, OSError, TypeError, ValueError) as error: + raise command_error(422, "invalid_studio_music_input", str(error)) from error + + +__all__ = ["prepare_studio_music"] diff --git a/app/services/studio_music_spec.py b/app/services/studio_music_spec.py new file mode 100644 index 000000000..5b50da734 --- /dev/null +++ b/app/services/studio_music_spec.py @@ -0,0 +1,503 @@ +"""Closed, provider-free contract for the Studio music command. + +Studio uses one native parameter map for image, speech and music. This +module freezes the music subset before model lookup, media inspection or task +admission. It keeps the literal lyrics and caption separate from the +effective native selectors and never calls the older ``freeze_music_spec``; +that helper intentionally normalizes Story requests and is not the Studio +snapshot authority. + +The versioned envelope is:: + + { + "version": 2, + "operation": "generation.music", + "intent_id": "...", + "input": { + "workspace": "...", + "workspace_collection_id": "...", + "params": {"model_type": "...", "prompt": "...", ...} + } + } + +Only the two local model IDs with a native music handler are registered in +this first vertical. Availability, model-specific defaults and resource +identity are checked by :mod:`studio_music_preparation`. +""" + +from __future__ import annotations + +from copy import deepcopy +import hashlib +import json +from typing import Annotated, Any, Literal + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StringConstraints, + StrictBool, + StrictFloat, + StrictInt, + StrictStr, + ValidationError, + field_validator, + model_validator, +) + +from services.image_generation_spec import ImageGenerationSpecError +from services.music_model_contract import ACE_DEFAULT, GUIDE_REVISION, MUSIC3_LOCAL +from services.studio_image_spec import _validate_reference + + +SCHEMA_VERSION = 2 +FINGERPRINT_VERSION = 2 +OPERATION = "generation.music" +STUDIO_MUSIC_SCHEMA_VERSION = SCHEMA_VERSION +STUDIO_MUSIC_OPERATION = OPERATION + +_MAX_ID_LENGTH = 240 +_MAX_INTENT_LENGTH = 160 +_MAX_PROMPT_LENGTH = 200_000 +_MAX_SHORT_TEXT_LENGTH = 8_192 +_MAX_LORA_COUNT = 64 + +# This is deliberately an exact registration. ``music_model_contract`` also +# knows remote/community IDs and ACE aliases for Story compatibility; this +# local command must never route one of those IDs to the generic music worker. +STUDIO_MUSIC_MODEL_TYPES = frozenset({ACE_DEFAULT, MUSIC3_LOCAL}) +MUSIC_MODEL_TYPES = STUDIO_MUSIC_MODEL_TYPES + +# These defaults are adapter-owned and contain no model-derived values. The +# preparation boundary fills duration, step count, guidance and custom model +# settings from the selected native definition. +STUDIO_MUSIC_DEFAULTS: dict[str, Any] = { + "generation_mode": "audio", + "_audio_sub_mode": "music", + "video_length": 0, + "image_mode": 0, + "multi_prompts_gen_type": 2, + "negative_prompt": "", + "repeat_generation": 1, + "batch_size": 1, + "activated_loras": [], + "loras_multipliers": "", + "audio_prompt_type": "", + "prompt_enhancer": "", + "_music_description": "", + "_music_instrumental": False, + "_tts_speaker_name1": "", + "_tts_speaker_name2": "", + "_tts_speaker_name3": "", + "_tts_speaker_name4": "", + "_tts_speaker_name5": "", + "_tts_speaker_name6": "", + "_tts_voice_count": 0, +} + +SUPPORTED_INPUT_FIELDS = ( + "prompt", + "alt_prompt", + "model_type", + "resolution", + "lyrics_language", + "video_length", + "num_inference_steps", + "guidance_scale", + "seed", + "image_mode", + "generation_mode", + "negative_prompt", + "repeat_generation", + "batch_size", + "activated_loras", + "loras_multipliers", + "audio_prompt_type", + "audio_guide", + "audio_guide2", + "audio_guide3", + "audio_guide4", + "audio_guide5", + "audio_guide6", + "audio_source", + "duration_seconds", + "temperature", + "top_p", + "top_k", + "audio_scale", + "alt_guidance_scale", + "guidance_phases", + "sample_solver", + "settings_version", + "prompt_enhancer", + "model_mode", + "custom_settings", + "multi_prompts_gen_type", + "_audio_sub_mode", + "_music_description", + "_music_instrumental", + "_tts_original_prompt", + "_tts_speaker_name1", + "_tts_speaker_name2", + "_tts_speaker_name3", + "_tts_speaker_name4", + "_tts_speaker_name5", + "_tts_speaker_name6", + "_tts_voice_count", +) + +# Shared Studio fields that are harmless only as sentinels are documented so +# a caller can project a form deliberately. They are not accepted as free +# command fields: the closed model below only accepts the explicitly typed +# music fields. +INACTIVE_MUSIC_FIELDS = ( + "image_mode=0", + "video_length=0", + "generation_mode=audio", + "_audio_sub_mode=music", + "multi_prompts_gen_type=2", + "repeat_generation=1", + "batch_size=1", + "negative_prompt=empty", + "prompt_enhancer=empty_or_null", + "_tts_speaker_name1..6=empty_or_null", + "_tts_voice_count=0", + "audio_guide3..6=empty_or_null", + "audio_source=empty_or_null", +) + +EXCLUDED_MUSIC_FIELDS = ( + "actor", + "client", + "permission", + "provenance", + "workspace inside input.params", + "filesystem paths", + "free-form provider payloads", + "remote MiniMax and community model IDs", + "speech voice names, voice counts and voice-clone controls", + "SFX/MMAudio, video, image, avatar and model3d controls", + "LLM song-writing or prompt enhancement", +) + + +class StudioMusicSpecError(ImageGenerationSpecError): + """Validation error for the closed Studio music envelope.""" + + +class _ClosedModel(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True, populate_by_name=False) + + +_Identity = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=_MAX_ID_LENGTH), +] +_Workspace = Annotated[ + StrictStr, + StringConstraints( + min_length=1, + max_length=_MAX_ID_LENGTH, + pattern=r"^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$", + ), +] +_WorkspaceCollectionId = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=200), +] +_IntentId = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=_MAX_INTENT_LENGTH), +] +_Prompt = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=_MAX_PROMPT_LENGTH), +] +_Text = Annotated[StrictStr, StringConstraints(max_length=_MAX_PROMPT_LENGTH)] +_ShortText = Annotated[StrictStr, StringConstraints(max_length=_MAX_SHORT_TEXT_LENGTH)] +_Name = Annotated[StrictStr, StringConstraints(min_length=1, max_length=_MAX_SHORT_TEXT_LENGTH)] +_Steps = Annotated[StrictInt, Field(ge=0, le=1000)] +_Seed = Annotated[StrictInt, Field(ge=-(2**63), le=2**63 - 1)] +_One = Annotated[StrictInt, Field(ge=1, le=1)] +_Two = Annotated[StrictInt, Field(ge=2, le=2)] +_Zero = Annotated[StrictInt, Field(ge=0, le=0)] +_PhaseCount = Annotated[StrictInt, Field(ge=0, le=16)] +_NonNegativeFinite = Annotated[StrictFloat, Field(ge=0, allow_inf_nan=False)] +_Guidance = Annotated[StrictFloat, Field(ge=0, le=1000, allow_inf_nan=False)] +_Unit = Annotated[StrictFloat, Field(ge=0, le=1, allow_inf_nan=False)] + + +class StudioMusicCustomSettings(_ClosedModel): + """Closed ACE-Step custom setting IDs shared by the native handler.""" + + bpm: Annotated[StrictInt, Field(ge=30, le=300)] | Literal[""] | None = None + keyscale: _ShortText | None = None + timesignature: Literal[2, 3, 4, 6, ""] | None = None + language: _ShortText | None = None + + +def _non_blank(value: str, field: str) -> None: + if not value.strip(): + raise ValueError(f"{field} must contain a non-blank value") + + +def _check_active_tts_metadata(params: Any) -> None: + for index in range(1, 7): + if getattr(params, f"tts_speaker_name{index}") not in (None, ""): + raise ValueError("TTS voice names are inactive in music mode") + if params.tts_voice_count != 0: + raise ValueError("input.params._tts_voice_count must be zero in music mode") + if params.tts_original_prompt is None and "tts_original_prompt" in params.model_fields_set: + raise ValueError("_tts_original_prompt must be a string when supplied") + + +class StudioMusicParams(_ClosedModel): + """Strict native music parameters emitted by Studio.""" + + prompt: _Prompt + alt_prompt: _Text = "" + model_type: _Identity + # Resolution is carried by the shared Studio form but is not interpreted + # by either native music handler. If present, it remains literal. + resolution: _ShortText | None = None + lyrics_language: _ShortText | None = None + + video_length: _Zero = 0 + num_inference_steps: _Steps | None = None + guidance_scale: _Guidance | None = None + seed: _Seed = -1 + image_mode: _Zero = 0 + generation_mode: Literal["audio"] = "audio" + negative_prompt: Literal["", None] = "" + repeat_generation: _One = 1 + batch_size: _One = 1 + activated_loras: list[_Name] = Field(default_factory=list, max_length=_MAX_LORA_COUNT) + loras_multipliers: _ShortText = "" + multi_prompts_gen_type: _Two = 2 + + # ACE-Step 1.5 exposes these exact source selectors. MiniMax-Music3 + # rejects all non-empty values during model preflight. + audio_prompt_type: Literal["", "A", "B", "AB"] = "" + audio_guide: StrictStr | None = None + audio_guide2: StrictStr | None = None + audio_guide3: StrictStr | None = None + audio_guide4: StrictStr | None = None + audio_guide5: StrictStr | None = None + audio_guide6: StrictStr | None = None + audio_source: Literal["", None] = None + + duration_seconds: _NonNegativeFinite | None = None + temperature: _NonNegativeFinite | None = None + top_p: _Unit | None = None + top_k: _Steps | None = None + audio_scale: _Guidance | None = None + alt_guidance_scale: _Guidance | None = None + guidance_phases: _PhaseCount | None = None + sample_solver: _ShortText = "" + settings_version: _NonNegativeFinite | None = None + prompt_enhancer: Literal["", None] = "" + model_mode: StrictInt | None = None + custom_settings: StudioMusicCustomSettings | None = None + + # Music UI metadata. ``_music_description`` is not used as a fallback + # for the caption, and ``_music_instrumental`` never rewrites lyrics. + music_description: _Text = Field("", alias="_music_description") + music_instrumental: StrictBool = Field(False, alias="_music_instrumental") + + # Studio currently serializes TTS bookkeeping for the shared audio tab. + # It is retained only as inactive metadata; active voice selection fails + # closed above rather than turning a music command into speech. + audio_sub_mode: Literal["music"] = Field("music", alias="_audio_sub_mode") + tts_original_prompt: _Text | None = Field(None, alias="_tts_original_prompt") + tts_speaker_name1: _Text | None = Field("", alias="_tts_speaker_name1") + tts_speaker_name2: _Text | None = Field("", alias="_tts_speaker_name2") + tts_speaker_name3: _Text | None = Field("", alias="_tts_speaker_name3") + tts_speaker_name4: _Text | None = Field("", alias="_tts_speaker_name4") + tts_speaker_name5: _Text | None = Field("", alias="_tts_speaker_name5") + tts_speaker_name6: _Text | None = Field("", alias="_tts_speaker_name6") + tts_voice_count: _Zero = Field(0, alias="_tts_voice_count") + + @field_validator( + "audio_guide", + "audio_guide2", + "audio_guide3", + "audio_guide4", + "audio_guide5", + "audio_guide6", + ) + @classmethod + def _check_audio_reference(cls, value): + if value in (None, ""): + return value + return _validate_reference(value) + + @field_validator("activated_loras") + @classmethod + def _check_lora_names(cls, values): + for value in values: + if not value.strip() or value in {".", ".."} or "/" in value or "\\" in value: + raise ValueError("activated_loras must contain exact catalog names") + return values + + @model_validator(mode="after") + def _check_semantics(self): + _non_blank(self.prompt, "input.params.prompt") + # ACE-Step accepts lyrics without a style caption. MiniMax requires + # one; reject that request here, before resources or task admission. + if self.model_type == MUSIC3_LOCAL: + _non_blank(self.alt_prompt, "input.params.alt_prompt") + _non_blank(self.model_type, "input.params.model_type") + if self.model_type not in STUDIO_MUSIC_MODEL_TYPES: + raise ValueError("input.params.model_type is not a registered local music model") + _check_active_tts_metadata(self) + return self + + +class StudioMusicInput(_ClosedModel): + workspace: _Workspace + workspace_collection_id: _WorkspaceCollectionId | None = None + params: StudioMusicParams + + @model_validator(mode="after") + def _check_collection_id(self): + if self.workspace_collection_id is not None: + _non_blank(self.workspace_collection_id, "input.workspace_collection_id") + return self + + +class _StudioMusicEnvelope(_ClosedModel): + version: Literal[SCHEMA_VERSION] + operation: Literal[OPERATION] + intent_id: _IntentId + input: StudioMusicInput + + @model_validator(mode="after") + def _check_intent(self): + _non_blank(self.intent_id, "intent_id") + return self + + +def _validation_error(exc: ValidationError) -> StudioMusicSpecError: + details: list[dict[str, Any]] = [] + messages: list[str] = [] + for error in exc.errors(include_input=False): + location = ".".join(str(part) for part in error.get("loc", ())) or "command" + message = str(error.get("msg") or "Invalid value") + details.append({"loc": location, "message": message, "type": error.get("type")}) + messages.append(f"{location}: {message}") + return StudioMusicSpecError( + "; ".join(messages) or "Invalid Studio music generation command", + details=details, + ) + + +def _canonical_content(effective: dict[str, Any]) -> dict[str, Any]: + return { + "version": SCHEMA_VERSION, + "operation": OPERATION, + "input": deepcopy(effective["input"]), + } + + +def _fingerprint(content: dict[str, Any]) -> str: + encoded = json.dumps( + content, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def freeze_studio_music_spec(command: Any) -> dict[str, Any]: + """Validate and detach one Studio music command without side effects.""" + if type(command) is not dict: + raise StudioMusicSpecError("Studio music generation command must be an object") + try: + envelope = _StudioMusicEnvelope.model_validate(command) + except ValidationError as exc: + raise _validation_error(exc) from exc + + original = deepcopy(command) + explicit_params = envelope.input.params.model_dump( + mode="json", by_alias=True, exclude_unset=True + ) + effective_params = deepcopy(explicit_params) + for key, value in STUDIO_MUSIC_DEFAULTS.items(): + effective_params.setdefault(key, deepcopy(value)) + # TTS bookkeeping is metadata only. When Studio omitted it, retain the + # exact lyrics literal as the native bookkeeping value without rewriting + # either prompt field. + effective_params.setdefault("_tts_original_prompt", explicit_params["prompt"]) + + effective = { + "version": SCHEMA_VERSION, + "operation": OPERATION, + "intent_id": envelope.intent_id, + "input": { + "workspace": envelope.input.workspace, + "params": effective_params, + }, + } + explicit_input = envelope.input.model_dump(mode="json", exclude_unset=True) + if "workspace_collection_id" in explicit_input: + effective["input"]["workspace_collection_id"] = explicit_input["workspace_collection_id"] + + return { + "original": original, + "effective": effective, + "fingerprint_version": FINGERPRINT_VERSION, + "fingerprint": _fingerprint(_canonical_content(effective)), + } + + +def studio_music_schema() -> dict[str, Any]: + """Return the discovery schema for the implemented local music boundary.""" + input_schema = StudioMusicInput.model_json_schema() + envelope_schema = _StudioMusicEnvelope.model_json_schema() + return { + "version": SCHEMA_VERSION, + "operation": OPERATION, + "intent_id": envelope_schema["properties"]["intent_id"], + "input": input_schema, + "supported_input_fields": list(SUPPORTED_INPUT_FIELDS), + "music_model_types": sorted(STUDIO_MUSIC_MODEL_TYPES), + "local_models": sorted(STUDIO_MUSIC_MODEL_TYPES), + "guide_revision": GUIDE_REVISION, + "effects": deepcopy(STUDIO_MUSIC_DEFAULTS), + "inactive": list(INACTIVE_MUSIC_FIELDS), + "excluded": list(EXCLUDED_MUSIC_FIELDS), + } + + +# Names used by the generic operation adapter and by discovery scripts. +StudioMusicGenerationInput = StudioMusicInput +StudioMusicGenerationParams = StudioMusicParams +music_generation_schema_v2 = studio_music_schema + + +__all__ = [ + "EXCLUDED_MUSIC_FIELDS", + "FINGERPRINT_VERSION", + "GUIDE_REVISION", + "INACTIVE_MUSIC_FIELDS", + "MUSIC_MODEL_TYPES", + "OPERATION", + "SCHEMA_VERSION", + "STUDIO_MUSIC_OPERATION", + "STUDIO_MUSIC_DEFAULTS", + "STUDIO_MUSIC_MODEL_TYPES", + "STUDIO_MUSIC_SCHEMA_VERSION", + "SUPPORTED_INPUT_FIELDS", + "StudioMusicCustomSettings", + "StudioMusicGenerationInput", + "StudioMusicGenerationParams", + "StudioMusicInput", + "StudioMusicParams", + "StudioMusicSpecError", + "freeze_studio_music_spec", + "music_generation_schema_v2", + "studio_music_schema", +] diff --git a/app/services/studio_sfx_commands.py b/app/services/studio_sfx_commands.py new file mode 100644 index 000000000..9a6154c00 --- /dev/null +++ b/app/services/studio_sfx_commands.py @@ -0,0 +1,47 @@ +"""Connect SFX to the existing native facade, admission and generation FIFO.""" +from copy import deepcopy +from pathlib import Path + +from routers.studio_sfx_commands import sfx_command_catalog +from services.native_generation_operation import NativeGenerationOperation +from services.studio_sfx_preparation import prepare_studio_sfx +from services.studio_sfx_resources import missing_mmaudio_files, validate_mmaudio_files +from services.studio_sfx_spec import SFX_MODEL_VARIANTS, freeze_studio_sfx_spec + + +def sfx_file_exists(runtime, filename): + """Use the same installed-file locator as MMAudio, without creating paths.""" + locator = getattr(getattr(runtime["wgp"], "fl", None), "locate_file", None) + if not callable(locator): + raise ValueError("The installed MMAudio file locator is unavailable") + path = locator(filename, error_if_none=False) + return bool(path and Path(path).is_file() and Path(path).stat().st_size > 0) + + +def check_sfx_models(runtime, variant): + return validate_mmaudio_files(variant, lambda filename: sfx_file_exists(runtime, filename)) + + +def create_sfx_operation(runtime, *, resources, execution_policy): + def freeze(command): + frozen = freeze_studio_sfx_spec(command) + effective = frozen["effective"]["input"] + return frozen, {**deepcopy(effective["params"]), "workspace": effective["workspace"]} + + def model_downloaded(model_type): + return not missing_mmaudio_files(model_type, lambda filename: sfx_file_exists(runtime, filename)) + + def prepare(params): + return prepare_studio_sfx( + params, model_definition={ + name: {"model_type": name, "architecture": "mmaudio", "variant": variant} + for name, variant in SFX_MODEL_VARIANTS.items() + }, model_downloaded=model_downloaded, resources=resources(), execution_policy=execution_policy, + ) + + return NativeGenerationOperation( + freeze=freeze, prepare=prepare, catalog=sfx_command_catalog(), + # Selecting a registered worker first verifies the canonical admission + # in ImageGenerationCommands.native_worker, including during recovery. + worker=runtime["_run_generation"], use_generation_defaults=False, + ) diff --git a/app/services/studio_sfx_execution.py b/app/services/studio_sfx_execution.py new file mode 100644 index 000000000..c7062a977 --- /dev/null +++ b/app/services/studio_sfx_execution.py @@ -0,0 +1,59 @@ +"""Recheck an admitted SFX request immediately before native model work. + +TaskRegistry owns both the receipt and its prepared snapshot. This check is +read-only and never downloads weights or creates another execution record. +""" +from collections.abc import Mapping +import json + +from services.studio_image_resources import file_identity + + +def _check_guide_identity(params, resources): + """A selected guide must retain its admitted bytes and identity.""" + guides = [resource for resource in resources + if resource.get("role") == "video_guide"] + path = params.get("video_guide") + if path: + if len(guides) != 1: + raise ValueError("The admitted SFX video has no unique resource identity") + try: + current = file_identity(path) + except OSError as error: + raise ValueError("The admitted SFX video is no longer available") from error + if any(current[key] != guides[0].get(key) for key in ("sha256", "size_bytes")): + raise ValueError("The admitted SFX video changed; select it in a new request") + elif guides: + raise ValueError("The admitted SFX video must not become a text-only request") + + +def prepared_sfx_execution(job, params, *, registry, check_models): + """Return whether this is a verified command, or reject changed resources. + + Ordinary legacy jobs return False. Claiming the typed capability requires + an existing matching admission; client provenance alone grants nothing. + """ + provenance = job.get("provenance") + if not isinstance(provenance, Mapping) or provenance.get("capability") != "generation.sfx": + return False + command = provenance.get("command") + intent = command.get("command_id") if isinstance(command, Mapping) else None + if not isinstance(intent, str) or not intent: + raise ValueError("SFX execution requires its original command receipt") + entry = registry.command_admission(intent) + if not isinstance(entry, Mapping) or entry.get("operation") != "generation.sfx": + raise ValueError("SFX execution has no matching command admission") + task = registry.get(entry["task_id"]) + if (not task or task["id"] != job.get("task_id") + or task.get("backend_job_id") != job.get("id") + or task.get("workspace") != job.get("workspace")): + raise ValueError("SFX execution does not match its admitted task") + runtime = entry["effective"]["runtime"] + expected = runtime["params"] + if (runtime["workspace"] != job.get("workspace") + or json.dumps(params, sort_keys=True, allow_nan=False) + != json.dumps(expected, sort_keys=True, allow_nan=False)): + raise ValueError("SFX execution parameters changed after admission") + _check_guide_identity(params, entry["effective"].get("resources", [])) + check_models(params["_mmaudio_variant"]) + return True diff --git a/app/services/studio_sfx_preparation.py b/app/services/studio_sfx_preparation.py new file mode 100644 index 000000000..afa1c9a5e --- /dev/null +++ b/app/services/studio_sfx_preparation.py @@ -0,0 +1,172 @@ +"""Provider-free preparation for the closed Studio SFX command. + +This boundary performs model catalog and installed-file checks through +injected callbacks, then asks :class:`StudioSfxResources` to resolve an +optional canonical video guide. It never downloads MMAudio files, schedules a +task or mutates the submitted parameters. The returned path-bearing mapping +is an internal handoff to the already-admitted native worker; resource +identities remain portable and are suitable for the durable receipt. +""" + +from __future__ import annotations + +from copy import deepcopy +import math +from collections.abc import Callable, Mapping +from typing import Any + +from fastapi import HTTPException + +from services.image_generation_commands import command_error +from services.studio_sfx_spec import SFX_MODEL_TYPES, SFX_MODEL_VARIANTS + + +def _definition_for(model_definition, model_type: str) -> dict[str, Any]: + if callable(model_definition): + definition = model_definition(model_type) + elif isinstance(model_definition, Mapping): + definition = model_definition.get(model_type) + else: + definition = None + if not isinstance(definition, Mapping): + raise ValueError("Choose an installed MMAudio SFX model from the model catalog") + return deepcopy(dict(definition)) + + +def _validate_model_definition( + model_type: str, + definition: Mapping[str, Any], + model_downloaded: Callable[[str], bool], +) -> str: + if model_type not in SFX_MODEL_TYPES: + raise ValueError("Choose a registered MMAudio SFX model") + expected_variant = SFX_MODEL_VARIANTS[model_type] + declared_type = definition.get("model_type") + if declared_type is not None and declared_type != model_type: + raise ValueError("The selected MMAudio model definition does not match its model ID") + declared_variant = definition.get("mmaudio_variant", definition.get("variant")) + if declared_variant is not None and declared_variant != expected_variant: + raise ValueError("The selected MMAudio model definition does not match its variant") + architecture = definition.get("architecture") + if architecture is not None and str(architecture).lower() not in {"mmaudio", "mmaudio_sfx", "sfx"}: + raise ValueError("The selected model definition is not an MMAudio SFX handler") + if not callable(model_downloaded) or not model_downloaded(model_type): + raise command_error( + 409, + "model_unavailable", + "Required MMAudio files are not installed; install them before submitting", + ) + return expected_variant + + +def _finite_duration(value: Any, *, video: bool) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError("input.params.duration_seconds must be a finite number") + duration = float(value) + if not math.isfinite(duration) or duration <= 0: + raise ValueError("input.params.duration_seconds must be greater than zero") + if not video and duration > 20: + raise ValueError("text-only SFX duration_seconds must be at most 20 seconds") + return duration + + +def _resource_duration(resources: list[dict[str, Any]]) -> float: + guides = [item for item in resources if item.get("role") == "video_guide"] + if len(guides) != 1: + raise ValueError("A selected SFX video guide must produce one resource identity") + duration = guides[0].get("duration_seconds") + if isinstance(duration, bool) or not isinstance(duration, (int, float)): + raise ValueError("A selected SFX video guide has no inspected duration") + duration = float(duration) + if not math.isfinite(duration) or duration <= 0: + raise ValueError("A selected SFX video guide has no finite positive duration") + return duration + + +def _prepare_media_duration(working, resources): + """Resolve guide timing without losing the requested duration control.""" + guide_selected = working.get("video_guide") not in (None, "") + requested_duration = _finite_duration( + working.get("duration_seconds"), video=guide_selected + ) + prepared, media = resources.prepare_media(working) + if not isinstance(prepared, dict) or not isinstance(media, list): + raise ValueError("SFX resource preparation returned invalid native parameters") + prepared = deepcopy(prepared) + media = deepcopy(media) + if guide_selected: + effective_duration = _resource_duration(media) + if prepared.get("video_guide") in (None, ""): + raise ValueError("A selected SFX video guide was not preserved by resource preparation") + prepared["duration_source"] = "video" + prepared["duration_seconds_requested"] = requested_duration + prepared["duration_seconds_effective"] = effective_duration + # MMAudio's video path derives the actual duration from the guide. + prepared["duration_seconds"] = effective_duration + else: + if prepared.get("video_guide") not in (None, ""): + raise ValueError("Text-only SFX preparation unexpectedly retained a video guide") + prepared["duration_source"] = "text" + prepared["duration_seconds_requested"] = requested_duration + prepared["duration_seconds_effective"] = requested_duration + prepared["duration_seconds"] = requested_duration + + return prepared, media + + +def prepare_studio_sfx( + params, + *, + model_definition, + model_downloaded, + resources, + execution_policy, +): + """Return detached native MMAudio parameters and portable identities.""" + if not isinstance(params, dict): + raise command_error(422, "invalid_studio_sfx_input", "SFX parameters must be an object") + working = deepcopy(params) + try: + workspace = working.get("workspace") + if not isinstance(workspace, str) or not workspace.strip(): + raise ValueError("input.workspace must be an explicit output workspace") + if not callable(execution_policy): + raise TypeError("execution_policy must be a workspace policy callback") + execution_policy(workspace) + + model_type = working.get("model_type") + if not isinstance(model_type, str): + raise ValueError("input.params.model_type must be a string") + definition = _definition_for(model_definition, model_type) + variant = _validate_model_definition(model_type, definition, model_downloaded) + + prepared, media = _prepare_media_duration(working, resources) + + # These values select the already-registered native MMAudio worker. + # They are generated here after model/resource checks, never trusted + # from a caller as authority or a download permission. + prepared["_mmaudio_variant"] = variant + prepared["MMAudio_setting"] = 1 + prepared["sfx_mode"] = True + prepared["generation_mode"] = "audio" + prepared["_audio_sub_mode"] = "sfx" + prepared["image_mode"] = 0 + prepared["video_length"] = 0 + prepared.setdefault("num_inference_steps", 25) + prepared.setdefault("guidance_scale", 4.5) + prepared.setdefault("seed", -1) + prepared.setdefault("MMAudio_neg_prompt", "") + prepared.setdefault("sfx_text_weight", 1.0) + positive = prepared.get("MMAudio_prompt") or prepared.get("prompt") + if not isinstance(positive, str) or not positive.strip(): + raise ValueError("input.params.prompt must contain a non-blank value") + prepared["prompt"] = prepared.get("prompt") or positive + prepared["MMAudio_prompt"] = prepared.get("MMAudio_prompt") or positive + return prepared, media + except HTTPException: + raise + except (OSError, TypeError, ValueError) as error: + raise command_error(422, "invalid_studio_sfx_input", str(error)) from error + + +__all__ = ["prepare_studio_sfx"] diff --git a/app/services/studio_sfx_resources.py b/app/services/studio_sfx_resources.py new file mode 100644 index 000000000..9d6a294de --- /dev/null +++ b/app/services/studio_sfx_resources.py @@ -0,0 +1,161 @@ +"""Inspect canonical video guides and installed MMAudio dependencies. + +SFX is the first Studio operation whose optional guide is a video. The +resolver intentionally reuses the image resource resolver's source-location +rules, but probes the selected file as video before the native worker sees it. +Only portable identities are returned in the command record; the resolved path +is kept in the detached worker map and is never serialized as provenance. +""" + +from __future__ import annotations + +from copy import deepcopy +import math +import subprocess +from collections.abc import Callable + +from services.studio_image_resources import StudioImageResources, file_identity +from services.studio_sfx_spec import SFX_MODEL_TYPES, SFX_MODEL_VARIANTS +from services.video_editor import probe_media + + +SFX_VIDEO_FIELDS = ("video_guide",) + +# These are the files opened by ``postprocessing.mmaudio`` for the two typed +# variants. The paths are relative names understood by the trusted model +# installer/locator. This list is only an inspection contract: no function in +# this module downloads, creates or mutates any of them. +MMAUDIO_SHARED_FILES = ( + "mmaudio/synchformer_state_dict.pth", + "mmaudio/v1-44.pth", + "DFN5B-CLIP-ViT-H-14-378/open_clip_config.json", + "DFN5B-CLIP-ViT-H-14-378/open_clip_pytorch_model.bin", + "bigvgan_v2_44khz_128band_512x/config.json", + "bigvgan_v2_44khz_128band_512x/bigvgan_generator.pt", +) +MMAUDIO_VARIANT_FILES = { + "v2": ("mmaudio/mmaudio_large_44k_v2.pth",), + "nsfw": ("mmaudio/mmaudio_large_44k_nsfw_gold_8.5k_final_fp16.safetensors",), +} + + +def required_mmaudio_files(model_or_variant: str) -> tuple[str, ...]: + """Return the trusted relative files needed by one typed MMAudio variant. + + Callers may provide the public virtual model ID or its derived variant. + Unknown values fail closed instead of selecting a server-configured + fallback model. + """ + if model_or_variant in SFX_MODEL_TYPES: + variant = SFX_MODEL_VARIANTS[model_or_variant] + elif model_or_variant in MMAUDIO_VARIANT_FILES: + variant = model_or_variant + else: + raise ValueError("Choose a registered MMAudio SFX variant") + return (*MMAUDIO_SHARED_FILES, *MMAUDIO_VARIANT_FILES[variant]) + + +def missing_mmaudio_files( + model_or_variant: str, + file_exists: Callable[[str], bool], +) -> tuple[str, ...]: + """Return missing dependency names using an injected read-only check.""" + if not callable(file_exists): + raise TypeError("file_exists must be a callable installed-file inspection") + required = required_mmaudio_files(model_or_variant) + missing: list[str] = [] + for filename in required: + try: + present = file_exists(filename) + except (OSError, TypeError, ValueError) as error: + raise ValueError("Installed MMAudio files could not be inspected") from error + if not isinstance(present, bool): + raise ValueError("Installed-file inspection must return booleans") + if not present: + missing.append(filename) + return tuple(missing) + + +def validate_mmaudio_files( + model_or_variant: str, + file_exists: Callable[[str], bool], +) -> tuple[str, ...]: + """Fail closed if any MMAudio dependency is absent; never download it.""" + required = required_mmaudio_files(model_or_variant) + missing = missing_mmaudio_files(model_or_variant, file_exists) + if missing: + joined = ", ".join(missing) + raise ValueError(f"Required MMAudio files are not installed: {joined}") + return required + + +class StudioSfxResources(StudioImageResources): + """Resolve one canonical optional video guide for typed SFX.""" + + media_kind = "video" + + def prepare_media(self, params): + """Return a detached worker map and portable guide identities. + + A missing guide is valid for text-only SFX. Once a guide is selected, + resolution, confinement, media kind and timing are all mandatory; a + missing or unreadable source never falls back to the output workspace. + """ + working = deepcopy(params) + value = working.get("video_guide") + if value in (None, ""): + working["video_guide"] = None + return working, [] + if not isinstance(value, str): + raise ValueError("input.params.video_guide must be a canonical video reference") + + # ``_media`` enforces an explicit uploads/workspace root and checks + # that an asset ID resolves to one unique video location. + path, workspace = self._media(value) + identity = file_identity(path) + try: + information = probe_media(path) + except subprocess.TimeoutExpired as error: + raise ValueError("A selected SFX video guide could not be inspected in time") from error + except (OSError, TypeError, ValueError) as error: + raise ValueError("A selected SFX video guide is not a readable video") from error + + duration = information.get("duration") + width = information.get("width") + height = information.get("height") + if (isinstance(duration, bool) or not isinstance(duration, (int, float)) + or not math.isfinite(float(duration)) or float(duration) <= 0): + raise ValueError("A selected SFX video guide has no finite positive duration") + if type(width) is not int or width <= 0 or type(height) is not int or height <= 0: + raise ValueError("A selected SFX video guide has invalid dimensions") + + duration = float(duration) + # Keep the path only in the native handoff. Resource records are + # deliberately made of API identity and media measurements. + working["video_guide"] = path + record = { + "role": "video_guide", + "index": 0, + "url": value, + "workspace": workspace, + "duration_seconds": duration, + "width": width, + "height": height, + "fps": information.get("fps"), + "has_audio": information.get("has_audio"), + "pixel_format": information.get("pixel_format"), + "has_alpha": information.get("has_alpha"), + **identity, + } + return working, [record] + + +__all__ = [ + "MMAUDIO_SHARED_FILES", + "MMAUDIO_VARIANT_FILES", + "SFX_VIDEO_FIELDS", + "StudioSfxResources", + "missing_mmaudio_files", + "required_mmaudio_files", + "validate_mmaudio_files", +] diff --git a/app/services/studio_sfx_spec.py b/app/services/studio_sfx_spec.py new file mode 100644 index 000000000..19ed6f4ed --- /dev/null +++ b/app/services/studio_sfx_spec.py @@ -0,0 +1,458 @@ +"""Closed, provider-free contract for a Studio sound-effects command. + +Studio SFX is backed by the existing MMAudio worker. The worker accepts a +video path (or ``None``), a positive and negative text prompt, a seed, a +duration and a text-conditioning weight. This module freezes that small +native surface before model/resource lookup or task admission. + +The versioned envelope is:: + + { + "version": 2, + "operation": "generation.sfx", + "intent_id": "...", + "input": { + "workspace": "...", + "workspace_collection_id": "...", + "params": { + "model_type": "mmaudio_v2", + "prompt": "...", + "MMAudio_neg_prompt": "...", + "duration_seconds": 5, + "video_guide": "/api/v1/file/clip.mp4?workspace=source" + } + } + } + +``original`` is a detached copy of the submitted envelope. ``effective`` +contains only adapter-owned native sentinels and the derived MMAudio variant; +model defaults and media measurements are added by the pure preparation +boundary. Host paths, provider payloads and authority flags never belong to +the caller's envelope. A prepared worker map may contain an already-resolved +path, but that map is an internal handoff and is not the command snapshot. +""" + +from __future__ import annotations + +from copy import deepcopy +import hashlib +import json +from typing import Annotated, Any, Literal + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StringConstraints, + StrictBool, + StrictFloat, + StrictInt, + StrictStr, + ValidationError, + field_validator, + model_validator, +) + +from services.image_generation_spec import ImageGenerationSpecError +from services.studio_image_spec import _validate_reference + + +SCHEMA_VERSION = 2 +FINGERPRINT_VERSION = 2 +OPERATION = "generation.sfx" +STUDIO_SFX_SCHEMA_VERSION = SCHEMA_VERSION +STUDIO_SFX_OPERATION = OPERATION + +_MAX_ID_LENGTH = 240 +_MAX_INTENT_LENGTH = 160 +_MAX_PROMPT_LENGTH = 200_000 +_MAX_REFERENCE_LENGTH = 8_192 +_MAX_SEED = 2**63 - 1 +_MIN_SEED = -(2**63) + +# These are the two exact virtual model IDs exposed by the Studio SFX +# selector. The preparation layer maps them to the worker's model names and +# checks the complete installed-file set through a callback. +SFX_MODEL_TYPES = frozenset({"mmaudio_v2", "mmaudio_nsfw"}) +SFX_VARIANTS = frozenset({"v2", "nsfw"}) +SFX_MODEL_VARIANTS = {"mmaudio_v2": "v2", "mmaudio_nsfw": "nsfw"} + +# These defaults are deterministic adapter selectors. They do not describe +# model availability or model-derived timing; those facts belong to the +# preparation callback. ``MMAudio_setting`` and ``sfx_mode`` are internal +# worker markers added to the effective projection. A caller cannot use them +# to authorize execution or bypass the native admission path. +STUDIO_SFX_DEFAULTS: dict[str, Any] = { + "generation_mode": "audio", + "_audio_sub_mode": "sfx", + "image_mode": 0, + "video_length": 0, + "num_inference_steps": 25, + "guidance_scale": 4.5, + "seed": -1, + "MMAudio_neg_prompt": "", + "sfx_text_weight": 1.0, + "MMAudio_setting": 1, + "sfx_mode": True, +} + +SUPPORTED_INPUT_FIELDS = ( + "prompt", + "MMAudio_prompt", + "MMAudio_neg_prompt", + "model_type", + "_mmaudio_variant", + "duration_seconds", + "seed", + "guidance_scale", + "sfx_text_weight", + "video_guide", + "generation_mode", + "_audio_sub_mode", + "image_mode", + "video_length", + "num_inference_steps", + "MMAudio_setting", + "sfx_mode", +) + +INACTIVE_SFX_FIELDS = ( + "generation_mode=audio", + "_audio_sub_mode=sfx", + "image_mode=0", + "video_length=0", + "audio_source=empty_or_null", + "video_source=empty_or_null", + "video_mask=empty_or_null", + "MMAudio_setting=1 (server-derived marker)", + "sfx_mode=true (server-derived marker)", +) + +EXCLUDED_SFX_FIELDS = ( + "actor", + "client", + "permission", + "provenance", + "filesystem paths", + "remote URLs", + "free-form provider payloads", + "video carrier model selection", + "download controls", + "queue or recovery controls", +) + + +class StudioSfxSpecError(ImageGenerationSpecError): + """Validation error for the closed Studio SFX envelope.""" + + +class _ClosedModel(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True, populate_by_name=False) + + +_Identity = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=_MAX_ID_LENGTH), +] +_Workspace = Annotated[ + StrictStr, + StringConstraints( + min_length=1, + max_length=_MAX_ID_LENGTH, + pattern=r"^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$", + ), +] +_WorkspaceCollectionId = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=200), +] +_IntentId = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=_MAX_INTENT_LENGTH), +] +_Text = Annotated[StrictStr, StringConstraints(max_length=_MAX_PROMPT_LENGTH)] +_NonBlankText = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=_MAX_PROMPT_LENGTH), +] +_ShortText = Annotated[StrictStr, StringConstraints(max_length=256)] +_Reference = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=_MAX_REFERENCE_LENGTH), +] +_Seed = Annotated[StrictInt, Field(ge=_MIN_SEED, le=_MAX_SEED)] +_Duration = Annotated[ + StrictFloat, + Field(gt=0, allow_inf_nan=False), +] +_Guidance = Annotated[ + StrictFloat, + Field(ge=0, le=1000, allow_inf_nan=False), +] +_TextWeight = Annotated[ + StrictFloat, + Field(ge=0, le=5, allow_inf_nan=False), +] +_Steps = Annotated[StrictInt, Field(ge=25, le=25)] + + +def _validate_prompt_aliases(prompt: str | None, native: str | None) -> None: + if (prompt is None or not prompt.strip()) and (native is None or not native.strip()): + raise ValueError("prompt or MMAudio_prompt must contain a non-blank value") + if prompt is not None and native is not None and prompt != native: + raise ValueError("prompt and MMAudio_prompt must match when both are supplied") + + +class StudioSfxParams(_ClosedModel): + """Typed native SFX parameters emitted by Studio.""" + + # ``prompt`` is the shared Studio field. ``MMAudio_prompt`` is the + # worker-native spelling used by the existing SFX panel. Exactly one is + # enough; when both are supplied they must agree because the worker gives + # MMAudio_prompt precedence. + prompt: _Text | None = None + mmaudio_prompt: _Text | None = Field(None, alias="MMAudio_prompt") + mmaudio_negative_prompt: _Text = Field("", alias="MMAudio_neg_prompt") + + # Keep the catalog registration visible to MCP/UI discovery as an enum; + # accepting an arbitrary model ID here would hide a carrier-model fallback + # until the later semantic validator. + model_type: Literal["mmaudio_v2", "mmaudio_nsfw"] + # The virtual model ID is the authoritative selector. This optional + # legacy alias is accepted only when it agrees with that ID; it is derived + # again in ``effective`` and is never trusted as an execution permission. + mmaudio_variant: Literal["v2", "nsfw"] | None = Field(None, alias="_mmaudio_variant") + + # A text-only pass is limited to MMAudio's 20 second pass. For a video + # guide this value is the user's control/requested duration and is kept + # separate from the inspected guide duration during preparation. + duration_seconds: _Duration + seed: _Seed = -1 + guidance_scale: _Guidance = 4.5 + sfx_text_weight: _TextWeight = 1.0 + num_inference_steps: _Steps = 25 + + # The optional reference is an exact asset ID or canonical local API URL. + # Resource preparation resolves it and replaces it with a confined worker + # path only in the detached internal native map. + video_guide: _Reference | Literal["", None] = None + + # Shared Studio sentinels. Active values belong to other operations and + # fail closed instead of selecting a carrier model or bypassing preflight. + generation_mode: Literal["audio"] = "audio" + audio_sub_mode: Literal["sfx"] = Field("sfx", alias="_audio_sub_mode") + image_mode: Annotated[StrictInt, Field(ge=0, le=0)] = 0 + video_length: Annotated[StrictInt, Field(ge=0, le=0)] = 0 + + # These two fields are present in the old SFX form body. They are + # validated as inert markers, then recomputed by the adapter in effective + # so a JSON caller cannot authorize itself as a prepared/native request. + mmaudio_setting: Literal[1] | None = Field(None, alias="MMAudio_setting") + sfx_mode: StrictBool | None = None + + @field_validator("prompt", "mmaudio_prompt") + @classmethod + def _prompt_type(cls, value): + return value + + @field_validator("video_guide") + @classmethod + def _canonical_video_reference(cls, value): + if value in (None, ""): + return value + try: + return _validate_reference(value) + except ValueError as error: + raise ValueError(str(error)) from error + + @model_validator(mode="after") + def _check_semantics(self): + _validate_prompt_aliases(self.prompt, self.mmaudio_prompt) + if self.model_type not in SFX_MODEL_TYPES: + raise ValueError("model_type must be mmaudio_v2 or mmaudio_nsfw") + # MMAudio's text-only worker path caps its requested duration at 20 s. + # A video-guided request keeps its control value as provenance, while + # preparation replaces the worker duration with the inspected guide + # duration. Do not apply the text-only cap to that control value. + if self.video_guide in (None, "") and self.duration_seconds > 20: + raise ValueError("text-only SFX duration_seconds must be at most 20 seconds") + expected_variant = SFX_MODEL_VARIANTS[self.model_type] + if self.mmaudio_variant is not None and self.mmaudio_variant != expected_variant: + raise ValueError("_mmaudio_variant does not match model_type") + if self.mmaudio_setting not in (None, 1): + # Literal validation normally catches this; keep a semantic guard + # for alternate Pydantic error paths and future aliases. + raise ValueError("MMAudio_setting is an internal SFX marker") + if self.sfx_mode not in (None, True): + raise ValueError("sfx_mode is an internal SFX marker") + return self + + +class StudioSfxInput(_ClosedModel): + workspace: _Workspace + workspace_collection_id: _WorkspaceCollectionId | None = None + params: StudioSfxParams + + @model_validator(mode="after") + def _check_collection_id(self): + if self.workspace_collection_id is not None and not self.workspace_collection_id.strip(): + raise ValueError("workspace_collection_id must contain a non-blank value") + return self + + +class _StudioSfxEnvelope(_ClosedModel): + version: Literal[SCHEMA_VERSION] + operation: Literal[OPERATION] + intent_id: _IntentId + input: StudioSfxInput + + @model_validator(mode="after") + def _check_intent(self): + if not self.intent_id.strip(): + raise ValueError("intent_id must contain a non-blank value") + return self + + +def _validation_error(exc: ValidationError) -> StudioSfxSpecError: + details: list[dict[str, Any]] = [] + messages: list[str] = [] + for error in exc.errors(include_input=False): + location = ".".join(str(part) for part in error.get("loc", ())) or "command" + message = str(error.get("msg") or "Invalid value") + details.append({"loc": location, "message": message, "type": error.get("type")}) + messages.append(f"{location}: {message}") + return StudioSfxSpecError( + "; ".join(messages) or "Invalid Studio SFX generation command", + details=details, + ) + + +def _canonical_content(effective: dict[str, Any]) -> dict[str, Any]: + return { + "version": SCHEMA_VERSION, + "operation": OPERATION, + "input": deepcopy(effective["input"]), + } + + +def _fingerprint(content: dict[str, Any]) -> str: + encoded = json.dumps( + content, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def freeze_studio_sfx_spec(command: Any) -> dict[str, Any]: + """Validate and detach one SFX command without I/O or side effects.""" + if type(command) is not dict: + raise StudioSfxSpecError("Studio SFX generation command must be an object") + try: + envelope = _StudioSfxEnvelope.model_validate(command) + except ValidationError as exc: + raise _validation_error(exc) from exc + + original = deepcopy(command) + explicit_params = envelope.input.params.model_dump( + mode="json", by_alias=True, exclude_unset=True + ) + effective_params = deepcopy(explicit_params) + for key, value in STUDIO_SFX_DEFAULTS.items(): + effective_params.setdefault(key, deepcopy(value)) + + # The worker reads MMAudio_prompt first. Fill only the missing spelling; + # an explicitly supplied value remains byte-for-byte unchanged. + positive = effective_params.get("MMAudio_prompt") or effective_params.get("prompt") + if not isinstance(effective_params.get("prompt"), str) or not effective_params["prompt"].strip(): + effective_params["prompt"] = positive + if ( + not isinstance(effective_params.get("MMAudio_prompt"), str) + or not effective_params["MMAudio_prompt"].strip() + ): + effective_params["MMAudio_prompt"] = positive + model_type = effective_params["model_type"] + effective_params["_mmaudio_variant"] = SFX_MODEL_VARIANTS[model_type] + # Keep the requested control independent from the future guide duration. + effective_params.setdefault( + "duration_seconds_requested", effective_params["duration_seconds"] + ) + effective_params.setdefault("duration_seconds_effective", None) + effective_params.setdefault("duration_source", "video" if effective_params.get("video_guide") else "text") + + effective = { + "version": SCHEMA_VERSION, + "operation": OPERATION, + "intent_id": envelope.intent_id, + "input": { + "workspace": envelope.input.workspace, + "params": effective_params, + }, + } + explicit_input = envelope.input.model_dump(mode="json", exclude_unset=True) + if "workspace_collection_id" in explicit_input: + effective["input"]["workspace_collection_id"] = explicit_input["workspace_collection_id"] + + return { + "original": original, + "effective": effective, + "fingerprint_version": FINGERPRINT_VERSION, + "fingerprint": _fingerprint(_canonical_content(effective)), + } + + +def studio_sfx_schema() -> dict[str, Any]: + """Return the executable discovery schema for ``generation.sfx``.""" + input_schema = StudioSfxInput.model_json_schema() + envelope_schema = _StudioSfxEnvelope.model_json_schema() + return { + "version": SCHEMA_VERSION, + "operation": OPERATION, + "intent_id": envelope_schema["properties"]["intent_id"], + "input": input_schema, + "supported_input_fields": list(SUPPORTED_INPUT_FIELDS), + "sfx_model_types": sorted(SFX_MODEL_TYPES), + "variants": sorted(SFX_VARIANTS), + "effects": deepcopy(STUDIO_SFX_DEFAULTS), + "limits": { + "text_duration_seconds": {"exclusive_minimum": 0, "maximum": 20}, + "video_duration_seconds": "derived from inspected video_guide", + "video_requested_duration_seconds": {"exclusive_minimum": 0}, + }, + "inactive": list(INACTIVE_SFX_FIELDS), + "excluded": list(EXCLUDED_SFX_FIELDS), + } + + +# Adapter/discovery aliases: these names all point at the one closed schema. +StudioSfxGenerationInput = StudioSfxInput +StudioSfxGenerationParams = StudioSfxParams +sfx_generation_schema_v2 = studio_sfx_schema +freeze_studio_sfx_command = freeze_studio_sfx_spec + + +__all__ = [ + "EXCLUDED_SFX_FIELDS", + "FINGERPRINT_VERSION", + "INACTIVE_SFX_FIELDS", + "OPERATION", + "SCHEMA_VERSION", + "SFX_MODEL_TYPES", + "SFX_MODEL_VARIANTS", + "SFX_VARIANTS", + "STUDIO_SFX_OPERATION", + "STUDIO_SFX_SCHEMA_VERSION", + "STUDIO_SFX_DEFAULTS", + "SUPPORTED_INPUT_FIELDS", + "StudioSfxGenerationInput", + "StudioSfxGenerationParams", + "StudioSfxInput", + "StudioSfxParams", + "StudioSfxSpecError", + "freeze_studio_sfx_command", + "freeze_studio_sfx_spec", + "sfx_generation_schema_v2", + "studio_sfx_schema", +] diff --git a/app/services/studio_speech_preparation.py b/app/services/studio_speech_preparation.py new file mode 100644 index 000000000..5ff2738b1 --- /dev/null +++ b/app/services/studio_speech_preparation.py @@ -0,0 +1,510 @@ +"""Provider-free preparation for the closed Studio speech command. + +The preparation boundary validates only facts declared by the selected native +handler, checks that speech references and settings are coherent, and asks the +existing ``StudioSpeechResources`` object to inspect canonical media/LoRAs. +It returns detached native parameters and resource identities; it does not +load a model, schedule a worker or create a second queue. +""" + +from __future__ import annotations + +from copy import deepcopy +import math +import re +from typing import Any, Mapping + +from fastapi import HTTPException + +from services.image_generation_commands import command_error +from services.studio_image_resources import validate_lora_multipliers +from services.studio_speech_spec import SPEECH_MODEL_TYPES + + +_AUDIO_REFERENCE_FIELDS = tuple( + ["audio_guide"] + [f"audio_guide{index}" for index in range(2, 7)] +) +_AUDIO_MODE_FLAGS = frozenset({"N", "V"}) + + +def _definition_for(model_definition, model_type: str) -> dict[str, Any]: + if callable(model_definition): + definition = model_definition(model_type) + elif isinstance(model_definition, Mapping): + definition = model_definition.get(model_type) + else: + definition = None + if not isinstance(definition, dict): + raise ValueError("Choose an installed speech model from the model catalog") + return deepcopy(definition) + + +def _finite_number(value, field: str, *, minimum: float | None = None, + maximum: float | None = None) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"input.params.{field} must be a finite number") + parsed = float(value) + if not math.isfinite(parsed): + raise ValueError(f"input.params.{field} must be a finite number") + if minimum is not None and parsed < minimum: + raise ValueError(f"input.params.{field} is below the model's declared minimum") + if maximum is not None and parsed > maximum: + raise ValueError(f"input.params.{field} exceeds the model's declared maximum") + return parsed + + +def _choice_values(choices) -> list[str]: + if not isinstance(choices, (list, tuple)): + return [] + values: list[str] = [] + for item in choices: + if isinstance(item, Mapping): + value = item.get("value") + elif isinstance(item, (list, tuple)) and len(item) >= 2: + value = item[1] + else: + value = item + if isinstance(value, str): + values.append(value) + return values + + +def _model_mode(working: dict[str, Any], definition: dict[str, Any]) -> None: + mode = working.get("model_mode") + if mode is not None and not isinstance(mode, str): + raise ValueError("input.params.model_mode must be a string or null") + modes = definition.get("model_modes") + if not isinstance(modes, Mapping): + if mode not in (None, ""): + raise ValueError("input.params.model_mode is not supported by this speech model") + return + + choices = _choice_values(modes.get("choices")) + default = modes.get("default") + if mode in (None, ""): + if isinstance(default, str) and default != "": + if choices and default not in choices: + raise ValueError("the speech model advertises an invalid model_mode default") + working["model_mode"] = default + return + if choices and mode not in choices: + raise ValueError("input.params.model_mode is not one of the model's declared choices") + + +def _duration(working: dict[str, Any], definition: dict[str, Any]) -> None: + value = working.get("duration_seconds") + slider = definition.get("duration_slider") + if not isinstance(slider, Mapping): + if value is not None: + _finite_number(value, "duration_seconds", minimum=0) + return + if value is None: + # ``default=0`` is meaningful for DramaBox (auto duration), so test + # key presence rather than using truthiness. + for key in ("default", "max"): + if key in slider and slider[key] is not None: + value = slider[key] + break + if value is None: + value = 600 + working["duration_seconds"] = value + minimum = slider.get("min") + maximum = slider.get("max") + _finite_number(value, "duration_seconds", minimum=float(minimum) if minimum is not None else 0, + maximum=float(maximum) if maximum is not None else None) + + +def _validate_inference_steps(steps: Any, definition: dict[str, Any]) -> None: + if type(steps) is not int or steps < 0: + raise ValueError("input.params.num_inference_steps must be a non-negative integer") + # Most speech handlers expose no step control and use the zero sentinel. + # Scenema is the declared exception: its locked native profile forces its + # own fixed step count (currently eight) even though the control is not + # user-editable. Preserve that model-owned value for the native facade. + if definition.get("inference_steps") is False and steps != 0 and not definition.get("lock_inference_steps"): + raise ValueError("input.params.num_inference_steps must be zero for this speech model") + if definition.get("inference_steps") is not False: + lower = definition.get("inference_steps_min") + upper = definition.get("inference_steps_max") + if lower is not None and steps < int(lower): + raise ValueError("input.params.num_inference_steps is below the model's declared minimum") + if upper is not None and steps > int(upper): + raise ValueError("input.params.num_inference_steps exceeds the model's declared maximum") + + +def _sampling_phases(working: dict[str, Any], definition: dict[str, Any]): + if "guidance_phases" not in working: + declared_phases = definition.get("guidance_max_phases") + working["guidance_phases"] = int(declared_phases) if declared_phases is not None else 1 + phases = working.get("guidance_phases", 1) + if type(phases) is not int or not 0 <= phases <= 16: + raise ValueError("input.params.guidance_phases must be an integer from 0 to 16") + maximum = definition.get("guidance_max_phases") + if maximum is not None and phases > int(maximum): + raise ValueError("input.params.guidance_phases exceeds the model's declared maximum") + return maximum + + +def _validate_sampling_scalars(working: dict[str, Any], definition: dict[str, Any]) -> None: + solver = working.get("sample_solver", "") + choices = _choice_values(definition.get("sample_solvers")) + if solver and choices and solver not in choices: + raise ValueError("input.params.sample_solver is not one of the model's declared choices") + + if working.get("negative_prompt") and definition.get("no_negative_prompt"): + raise ValueError("input.params.negative_prompt is not supported by this speech model") + temperature = working.get("temperature") + if temperature is not None: + _finite_number(temperature, "temperature", minimum=0) + if definition.get("temperature") is False: + raise ValueError("input.params.temperature is locked for this speech model") + pause = working.get("pause_seconds") + if pause is not None: + _finite_number(pause, "pause_seconds", minimum=0, maximum=2) + if not definition.get("pause_between_sentences"): + raise ValueError("input.params.pause_seconds is not supported by this speech model") + + +def _validate_top_sampling_field( + field: str, value: Any, capability: str, definition: dict[str, Any] +) -> None: + if value is None: + return + if not definition.get(capability): + raise ValueError(f"input.params.{field} is not supported by this speech model") + if field == "top_p": + _finite_number(value, field, minimum=0, maximum=1) + elif type(value) is not int or value < 0: + raise ValueError("input.params.top_k must be a non-negative integer") + + +def _validate_sampling_options(working: dict[str, Any], definition: dict[str, Any]) -> None: + _validate_sampling_scalars(working, definition) + for field, capability in (("top_p", "top_p_slider"), ("top_k", "top_k_slider")): + _validate_top_sampling_field(field, working.get(field), capability, definition) + + +def _sampling(working: dict[str, Any], definition: dict[str, Any]) -> int: + steps = working.get("num_inference_steps", 0) + working.setdefault("num_inference_steps", 0) + _validate_inference_steps(steps, definition) + maximum = _sampling_phases(working, definition) + _validate_sampling_options(working, definition) + return max(1, int(maximum if maximum is not None else 1)) + + +def _strip_audio_modifier_flags( + mode: str, source: Mapping[str, Any] | None, + definition: Mapping[str, Any] | None, +) -> str: + custom_flags = source.get("custom_flags") if source else None + if isinstance(custom_flags, Mapping): + for flag in custom_flags: + if isinstance(flag, str) and flag: + mode = mode.replace(flag.upper(), "") + custom = definition.get("audio_prompt_type_custom_option") if definition else None + custom_flag = custom.get("flag") if isinstance(custom, Mapping) else None + if isinstance(custom_flag, str) and custom_flag: + mode = mode.replace(custom_flag.upper(), "") + return mode + + +def _base_audio_mode(value: str, source: Mapping[str, Any] | None, + definition: Mapping[str, Any] | None = None) -> str: + mode = value.upper() + mode_without_modifiers = "".join(char for char in mode if char not in _AUDIO_MODE_FLAGS) + # A mode already present in the native selection is a complete value + # (Scenema's A2/AB2 include the SeedVC ``2`` flag). Strip custom flags + # only when they are being used as standalone modifiers. + selections = _choice_values(source.get("selection")) if source else [] + if mode_without_modifiers in selections: + return mode_without_modifiers + mode = _strip_audio_modifier_flags(mode_without_modifiers, source, definition) + return "".join(char for char in mode if char not in _AUDIO_MODE_FLAGS) + + +def _voice_mode_for_count(count: int, selection: list[str]) -> str: + if not selection: + return "" + if count <= 0: + return selection[0] + if count == 1: + return selection[min(1, len(selection) - 1)] + return selection[min(2, len(selection) - 1)] + + +def _audio_mode_source( + definition: dict[str, Any], +) -> tuple[Mapping[str, Any] | None, list[str]]: + source = definition.get("audio_prompt_type_sources") + source = source if isinstance(source, Mapping) else None + selection = _choice_values(source.get("selection")) if source else [] + # Some model definitions use a plain string list for selection and the + # helper above deliberately handles only native list/tuple values. + if source and not selection and isinstance(source.get("selection"), list): + selection = [item for item in source["selection"] if isinstance(item, str)] + if source and not selection: + raise ValueError("the speech model advertises no audio prompt choices") + return source, selection + + +def _audio_mode_default( + working: dict[str, Any], raw_mode: str, source: Mapping[str, Any] | None +) -> str: + mode = raw_mode + if not mode and source and isinstance(source.get("default"), str): + mode = source["default"] + if mode: + working["audio_prompt_type"] = mode + return mode + + +def _speech_audio_mode( + working: dict[str, Any], definition: dict[str, Any] +) -> tuple[str, str, Mapping[str, Any] | None, list[str]]: + raw_mode = working.get("audio_prompt_type") or "" + if not isinstance(raw_mode, str): + raise ValueError("input.params.audio_prompt_type must be a string") + source, selection = _audio_mode_source(definition) + mode = _audio_mode_default(working, raw_mode, source) + base = _base_audio_mode(mode, source, definition) + return mode, base, source, selection + + +def _validate_declared_audio_mode( + mode: str, + base: str, + source: Mapping[str, Any] | None, + definition: dict[str, Any], + selection: list[str], +) -> None: + if source: + custom = definition.get("audio_prompt_type_custom_option") + custom_flag = custom.get("flag") if isinstance(custom, Mapping) else None + source_custom_flags = source.get("custom_flags") + has_source_custom = ( + isinstance(source_custom_flags, Mapping) + and any(isinstance(flag, str) and flag and flag.upper() in mode.upper() + for flag in source_custom_flags) + ) + if base not in selection: + # A custom-only flag (DramaBox's ``0``) has an empty base and is + # valid alongside the selected empty mode. + if not ( + not base + and ( + (isinstance(custom_flag, str) and custom_flag.upper() in mode.upper()) + or has_source_custom + ) + ): + raise ValueError("input.params.audio_prompt_type is not a declared model choice") + elif mode and mode not in {"A", "AN", "AV"}: + # Chatterbox exposes any_audio_prompt but no choice map; its native + # handler accepts its historical A selector and no other mode. + raise ValueError("input.params.audio_prompt_type is not supported by this speech model") + + +def _validate_voice_count( + working: dict[str, Any], definition: dict[str, Any], mode: str, + source: Mapping[str, Any] | None, selection: list[str], +) -> int: + count = working.get("_tts_voice_count", 0) + if type(count) is not int or not 0 <= count <= 6: + raise ValueError("input.params._tts_voice_count must be an integer from 0 to 6") + maximum_count = definition.get("max_voice_count") + if maximum_count is not None and count > int(maximum_count): + raise ValueError("input.params._tts_voice_count exceeds the model's voice limit") + + if definition.get("audio_mode_from_voice_count"): + expected = _voice_mode_for_count(count, selection) + if _base_audio_mode(mode, source, definition) != expected: + raise ValueError("input.params.audio_prompt_type must match the selected voice count") + return count + + +def _audio_reference_state( + working: dict[str, Any], definition: dict[str, Any], count: int +) -> tuple[dict[str, Any], int]: + refs = {field: working.get(field) for field in _AUDIO_REFERENCE_FIELDS} + highest_ref = 0 + for index, field in enumerate(_AUDIO_REFERENCE_FIELDS, start=1): + value = refs[field] + if value not in (None, ""): + highest_ref = index + if definition.get("audio_mode_from_voice_count") and count < index: + raise ValueError(f"input.params.{field} requires that voice slot to be selected") + if highest_ref and definition.get("audio_mode_from_voice_count") and count < highest_ref: + raise ValueError("audio references exceed the selected voice count") + return refs, highest_ref + + +def _validate_primary_audio_references(refs: dict[str, Any], base: str) -> None: + needs_first = "A" in base or base == "2" + needs_second = "B" in base or base == "2" + if needs_first and not refs["audio_guide"]: + raise ValueError("input.params.audio_prompt_type requires audio_guide") + if needs_second and not refs["audio_guide2"]: + raise ValueError("input.params.audio_prompt_type requires audio_guide2") + if refs["audio_guide"] and not needs_first: + raise ValueError("audio_guide is selected but audio_prompt_type has no first reference") + if refs["audio_guide2"] and not needs_second: + raise ValueError("audio_guide2 is selected but audio_prompt_type has no second reference") + + +def _validate_additional_audio_references(refs: dict[str, Any], base: str) -> None: + for index in range(3, 7): + if refs[f"audio_guide{index}"] and "B" not in base: + raise ValueError(f"audio_guide{index} requires a multi-voice audio_prompt_type") + + +def _validate_audio_reference_modes( + refs: dict[str, Any], base: str, source: Mapping[str, Any] | None +) -> None: + # Explicit model choices are the evidence that A/B references are active. + # Models without a choice map (Chatterbox) leave reference requirements to + # their native handler instead of this adapter inventing one. + if source: + _validate_primary_audio_references(refs, base) + _validate_additional_audio_references(refs, base) + + +def _validate_audio_prompts(working: dict[str, Any], definition: dict[str, Any]) -> None: + mode, base, source, selection = _speech_audio_mode(working, definition) + _validate_declared_audio_mode(mode, base, source, definition, selection) + count = _validate_voice_count(working, definition, mode, source, selection) + refs, _ = _audio_reference_state(working, definition, count) + _validate_audio_reference_modes(refs, base, source) + + +def _setting_id(setting: Mapping[str, Any], index: int) -> str: + explicit = setting.get("id") + if isinstance(explicit, str) and explicit.strip(): + return explicit.strip() + name = setting.get("name") + if isinstance(name, str) and name.strip(): + normalized = re.sub(r"[^a-z0-9_]+", "_", name.strip().lower()).strip("_") + if normalized: + return normalized + return f"custom_setting_{index + 1}" + + +def _custom_setting_definitions(definition: dict[str, Any]): + definitions = definition.get("custom_settings") + if not isinstance(definitions, list): + definitions = definition.get("custom_settings_def") + return definitions if isinstance(definitions, list) else None + + +def _validate_custom_setting_range(key: str, numeric: float | None, setting: Mapping[str, Any]) -> None: + if numeric is None: + return + lower = setting.get("min", setting.get("minimum")) + upper = setting.get("max", setting.get("maximum")) + if lower is not None and numeric < float(lower): + raise ValueError(f"input.params.custom_settings.{key} is below its declared minimum") + if upper is not None and numeric > float(upper): + raise ValueError(f"input.params.custom_settings.{key} exceeds its declared maximum") + + +def _validate_custom_setting(key: str, value: Any, setting: Mapping[str, Any]) -> None: + setting_type = str(setting.get("type", "text")).lower() + if value == "" and setting_type in {"int", "float"}: + # The Studio numeric control uses an empty string while a + # selectable setting is inactive; native handlers remove it. + return + if setting_type == "int": + if type(value) is not int: + raise ValueError(f"input.params.custom_settings.{key} must be an integer") + numeric = float(value) + elif setting_type == "float": + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"input.params.custom_settings.{key} must be a number") + numeric = float(value) + if not math.isfinite(numeric): + raise ValueError(f"input.params.custom_settings.{key} must be finite") + elif setting_type == "bool": + if type(value) is not bool: + raise ValueError(f"input.params.custom_settings.{key} must be boolean") + numeric = None + else: + if not isinstance(value, str): + raise ValueError(f"input.params.custom_settings.{key} must be text") + numeric = None + _validate_custom_setting_range(key, numeric, setting) + + +def _validate_custom_settings(working: dict[str, Any], definition: dict[str, Any]) -> None: + values = working.get("custom_settings") + if values is None: + return + if not isinstance(values, dict): + raise ValueError("input.params.custom_settings must be an object") + definitions = _custom_setting_definitions(definition) + if not isinstance(definitions, list): + if values: + raise ValueError("input.params.custom_settings is not supported by this speech model") + return + known = {_setting_id(item, index): item for index, item in enumerate(definitions) if isinstance(item, Mapping)} + unknown = sorted(set(values) - set(known)) + if unknown: + raise ValueError("input.params.custom_settings contains an unknown model setting") + for key, value in values.items(): + _validate_custom_setting(key, value, known[key]) + + +def _validate_loras(working: dict[str, Any], definition: dict[str, Any], phases: int) -> None: + names = working.get("activated_loras") or [] + if names and not definition.get("enabled_audio_lora"): + raise ValueError("selected LoRAs are not supported by this speech model") + if names or working.get("loras_multipliers"): + validate_lora_multipliers(working, phases) + + +def _validate_speech_model(model_type: str, definition: dict[str, Any], model_downloaded) -> None: + if model_type not in SPEECH_MODEL_TYPES: + raise ValueError("Choose a registered speech model; music and SFX models use another operation") + if definition.get("audio_only") is not True or definition.get("image_outputs"): + raise ValueError("The selected model is not an audio-only speech model") + if not model_downloaded(model_type): + raise command_error(409, "model_unavailable", "Required speech model files are not installed; install them before submitting") + + +def prepare_studio_speech(params, *, model_definition, model_downloaded, resources, + execution_policy): + """Return detached native speech parameters and inspected resources. + + ``params`` is normally the effective output of + :func:`freeze_studio_speech_spec`; accepting a mapping directly keeps this + boundary easy to test and lets the native facade supply its own snapshot. + """ + if not isinstance(params, dict): + raise command_error(422, "invalid_studio_speech_input", "Speech parameters must be an object") + working = deepcopy(params) + try: + workspace = working.get("workspace") + if not isinstance(workspace, str) or not workspace.strip(): + raise ValueError("input.workspace must be an explicit output workspace") + execution_policy(workspace) + model_type = working.get("model_type") + if not isinstance(model_type, str): + raise ValueError("input.params.model_type must be a string") + definition = _definition_for(model_definition, model_type) + _validate_speech_model(model_type, definition, model_downloaded) + phases = _sampling(working, definition) + _model_mode(working, definition) + _duration(working, definition) + _validate_audio_prompts(working, definition) + _validate_custom_settings(working, definition) + _validate_loras(working, definition, phases) + prepared, media = resources.prepare_media(working) + loras = resources.prepare_loras(working, definition) + if not isinstance(prepared, dict): + raise ValueError("speech resource preparation returned invalid native parameters") + return deepcopy(prepared), [*deepcopy(media), *deepcopy(loras)] + except HTTPException: + raise + except (OSError, ValueError) as error: + raise command_error(422, "invalid_studio_speech_input", str(error)) from error + + +__all__ = ["prepare_studio_speech"] diff --git a/app/services/studio_speech_resources.py b/app/services/studio_speech_resources.py new file mode 100644 index 000000000..7f2ab2d80 --- /dev/null +++ b/app/services/studio_speech_resources.py @@ -0,0 +1,35 @@ +"""Inspect canonical speech references without changing their source workspace.""" +from copy import deepcopy +import math +import subprocess + +from services.studio_image_resources import StudioImageResources, file_identity +from services.video_editor import probe_audio + + +SPEECH_AUDIO_FIELDS = ("audio_guide", *(f"audio_guide{i}" for i in range(2, 7))) + + +class StudioSpeechResources(StudioImageResources): + media_kind = "audio" + + def prepare_media(self, params): + working = deepcopy(params) + resources = [] + for field in SPEECH_AUDIO_FIELDS: + value = working.get(field) + if not value: + continue + path, workspace = self._media(value) + identity = file_identity(path) + try: + information = probe_audio(path) + except subprocess.TimeoutExpired as error: + raise ValueError("A selected voice reference could not be inspected in time") from error + duration = information["duration"] + if not math.isfinite(duration) or duration <= 0: + raise ValueError("A selected voice reference has no finite positive duration") + resources.append({"role": field, "index": 0, "url": value, + "workspace": workspace, "duration_seconds": duration, **identity}) + working[field] = path + return working, resources diff --git a/app/services/studio_speech_spec.py b/app/services/studio_speech_spec.py new file mode 100644 index 000000000..85d8b3932 --- /dev/null +++ b/app/services/studio_speech_spec.py @@ -0,0 +1,637 @@ +"""Closed, provider-free contract for the Studio speech command. + +The browser has one native parameter map for all Studio modes. A speech +command must freeze the speech part of that map before model lookup, resource +inspection, task admission or worker effects. This module owns that boundary; +the model handler remains the authority for provider-specific generation +validation. + +The v2 envelope is:: + + { + "version": 2, + "operation": "generation.speech", + "intent_id": "...", + "input": { + "workspace": "...", + "params": {"prompt": "...", "model_type": "...", ...} + } + } + +``original`` is an immutable detached copy of the submitted envelope. The +``effective`` map contains only deterministic adapter defaults; model-derived +defaults are applied by the preparation boundary. The fingerprint covers +the effective operation, workspace and native parameters, while excluding +the transport intent and caller metadata. + +Audio references are syntax-checked here and resolved by +``StudioSpeechResources`` later. No filesystem path, model catalog, provider +or provenance authority is consulted in this module. +""" + +from __future__ import annotations + +from copy import deepcopy +import hashlib +import json +import re +from typing import Annotated, Any, Literal + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StringConstraints, + StrictBool, + StrictFloat, + StrictInt, + StrictStr, + ValidationError, + field_validator, + model_validator, +) + +from services.image_generation_spec import ImageGenerationSpecError +from services.studio_image_spec import _validate_reference + + +SCHEMA_VERSION = 2 +FINGERPRINT_VERSION = 2 +OPERATION = "generation.speech" + +_MAX_ID_LENGTH = 240 +_MAX_INTENT_LENGTH = 160 +_MAX_PROMPT_LENGTH = 200_000 +_MAX_LORA_COUNT = 64 +_MAX_VOICE_COUNT = 6 + +# These are adapter-owned values. They describe the native selectors that +# make an audio submission unambiguous. Duration, model mode and model +# custom settings stay model-owned and are filled during preflight. +STUDIO_SPEECH_DEFAULTS: dict[str, Any] = { + "generation_mode": "audio", + "_audio_sub_mode": "speech", + "video_length": 0, + "image_mode": 0, + "multi_prompts_gen_type": 2, + "negative_prompt": "", + "repeat_generation": 1, + "activated_loras": [], + "loras_multipliers": "", + "audio_prompt_type": "", + "prompt_enhancer": "", + "minimax_h3_turbo_mode": False, + "_tts_speaker_name1": "", + "_tts_speaker_name2": "", + "_tts_speaker_name3": "", + "_tts_speaker_name4": "", + "_tts_speaker_name5": "", + "_tts_speaker_name6": "", + "_tts_voice_count": 0, +} + +# The allowlist is intentionally explicit. Several music and sound-effect +# handlers advertise ``audio_only`` and the same ``tts`` family, so that +# metadata alone cannot safely turn this speech operation into a music/SFX +# operation. New handlers need a deliberate registration here. +SPEECH_MODEL_TYPES = frozenset( + { + "kugelaudio_0_open", + "qwen3_tts_customvoice", + "qwen3_tts_voicedesign", + "qwen3_tts_base", + "chatterbox", + "index_tts2", + "scenema_audio", + "dramabox_audio", + } +) + +INACTIVE_SPEECH_FIELDS = ( + "image_mode", + "video_length", + "minimax_h3_turbo_mode", + "image_start", + "image_end", + "image_refs", + "image_guide", + "image_mask", + "video_guide", + "video_mask", + "video_source", + "audio_source", + "MMAudio_setting", + "MMAudio_prompt", + "MMAudio_neg_prompt", + "h3_ref_videos", + "h3_ref_audios", + "minimax_h3_references", + "spatial_upsampling", + "temporal_upsampling", + "wangp_processor_settings", +) + +EXCLUDED_SPEECH_FIELDS = ( + "actor", + "client", + "permission", + "provenance", + "workspace inside input.params", + "filesystem paths", + "free-form provider payloads", + "music and SFX model types", + "video, avatar and model3d controls", +) + +SUPPORTED_INPUT_FIELDS = ( + "prompt", + "alt_prompt", + "model_type", + "resolution", + "video_length", + "num_inference_steps", + "guidance_scale", + "seed", + "image_mode", + "generation_mode", + "negative_prompt", + "repeat_generation", + "activated_loras", + "loras_multipliers", + "audio_prompt_type", + "audio_guide", + "audio_guide2", + "audio_guide3", + "audio_guide4", + "audio_guide5", + "audio_guide6", + "audio_source", + "_audio_sub_mode", + "_tts_original_prompt", + "_tts_speaker_name1", + "_tts_speaker_name2", + "_tts_speaker_name3", + "_tts_speaker_name4", + "_tts_speaker_name5", + "_tts_speaker_name6", + "_tts_voice_count", + "duration_seconds", + "pause_seconds", + "temperature", + "top_p", + "top_k", + "audio_scale", + "audio_guidance_scale", + "alt_scale", + "guidance_phases", + "flow_shift", + "sample_solver", + "settings_version", + "prompt_enhancer", + "model_mode", + "custom_settings", + "tts_dynaudnorm", + "tts_comp_threshold", + "tts_comp_attack", + "tts_comp_release", + "tts_comp_makeup", + "multi_prompts_gen_type", + "minimax_h3_turbo_mode", + "image_start", + "image_end", + "image_refs", + "image_guide", + "image_mask", + "video_guide", + "video_mask", + "video_source", + "spatial_upsampling", + "temporal_upsampling", + "wangp_processor_settings", + "MMAudio_setting", + "MMAudio_prompt", + "MMAudio_neg_prompt", + "h3_ref_videos", + "h3_ref_audios", + "minimax_h3_references", +) + + +class StudioSpeechSpecError(ImageGenerationSpecError): + """Validation error for the closed Studio speech envelope.""" + + +class _ClosedModel(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True, populate_by_name=False) + + +_Identity = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=_MAX_ID_LENGTH), +] +_Workspace = Annotated[ + StrictStr, + StringConstraints( + min_length=1, + max_length=_MAX_ID_LENGTH, + pattern=r"^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$", + ), +] +_WorkspaceCollectionId = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=200), +] +_IntentId = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=_MAX_INTENT_LENGTH), +] +_Prompt = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=_MAX_PROMPT_LENGTH), +] +_Text = Annotated[StrictStr, StringConstraints(max_length=_MAX_PROMPT_LENGTH)] +_ShortText = Annotated[StrictStr, StringConstraints(max_length=8192)] +_ModelMode = Annotated[StrictStr, StringConstraints(max_length=256)] +_AudioPromptType = Annotated[ + StrictStr, + StringConstraints(max_length=32, pattern=r"^[A-Za-z0-9]*$"), +] +_NonNegativeInt = Annotated[StrictInt, Field(ge=0, le=100_000)] +_Count = Annotated[StrictInt, Field(ge=1, le=100)] +_VoiceCount = Annotated[StrictInt, Field(ge=0, le=_MAX_VOICE_COUNT)] +_Seed = Annotated[StrictInt, Field(ge=-(2**63), le=2**63 - 1)] +_Finite = Annotated[StrictFloat, Field(allow_inf_nan=False)] +_CustomFloat = Annotated[StrictFloat, Field(allow_inf_nan=False)] +_NonNegativeFinite = Annotated[ + StrictFloat, + Field(ge=0, allow_inf_nan=False), +] +_Guidance = Annotated[ + StrictFloat, + Field(ge=0, le=1000, allow_inf_nan=False), +] +_Zero = Annotated[StrictInt, Field(ge=0, le=0)] +_DialogueMode = Annotated[StrictInt, Field(ge=2, le=2)] +_TtsDynaudnorm = Annotated[StrictInt, Field(ge=0, le=1)] | StrictBool + + +class StudioSpeechCustomSettings(_ClosedModel): + """Typed union of custom settings exposed by speech handlers. + + The selected model still owns which subset is valid and its metadata + ranges. Keeping the known IDs closed here prevents arbitrary nested JSON + from crossing the command boundary while allowing each speech handler's + currently published setting family. + """ + + auto_split_every_s: _CustomFloat | Literal[""] | None = None + exaggeration: _CustomFloat | None = None + pace: _CustomFloat | None = None + vc_steps: StrictInt | None = None + vc_cfg_rate: _CustomFloat | None = None + duration_multiplier: _CustomFloat | None = None + + +def _check_required_speech_texts(params: Any) -> None: + for field in ("prompt", "model_type", "resolution"): + if not getattr(params, field).strip(): + raise ValueError(f"{field} must contain a non-blank value") + + +def _check_speech_prompt_metadata(params: Any) -> None: + if params.tts_original_prompt is None and "tts_original_prompt" in params.model_fields_set: + raise ValueError("_tts_original_prompt must be a string when supplied") + + +def _check_speech_ranges(params: Any) -> None: + if params.pause_seconds is not None and params.pause_seconds > 2: + raise ValueError("pause_seconds must be between 0 and 2 seconds") + if params.temperature is not None and params.temperature > 2: + raise ValueError("temperature must be at most 2") + if params.tts_comp_threshold is not None and not -50 <= params.tts_comp_threshold <= -10: + raise ValueError("tts_comp_threshold must be between -50 and -10") + if params.tts_comp_release is not None and params.tts_comp_release > 500: + raise ValueError("tts_comp_release must be at most 500") + if params.tts_comp_makeup is not None and params.tts_comp_makeup > 12: + raise ValueError("tts_comp_makeup must be at most 12") + + +class StudioSpeechParams(_ClosedModel): + """Typed native speech parameters emitted by Studio.""" + + prompt: _Prompt + alt_prompt: _Text = "" + model_type: _Identity + resolution: Annotated[StrictStr, StringConstraints(min_length=1, max_length=128)] + + # Speech jobs retain the common native selectors as explicit inactive + # sentinels. They are exact zero values, not a mode inferred from a + # missing image field. + video_length: _Zero = 0 + num_inference_steps: _NonNegativeInt = 0 + guidance_scale: _Guidance = 1.0 + seed: _Seed = -1 + image_mode: _Zero = 0 + generation_mode: Literal["audio"] = "audio" + negative_prompt: _Text = "" + repeat_generation: _Count = 1 + activated_loras: list[Annotated[StrictStr, StringConstraints(min_length=1, max_length=8192)]] = Field( + default_factory=list, max_length=_MAX_LORA_COUNT + ) + loras_multipliers: _ShortText = "" + multi_prompts_gen_type: _DialogueMode = 2 + + # Native speech selectors and canonical reference URLs/asset IDs. Empty + # string and null are retained as inactive sentinels; paths are rejected + # by the validator below and resolved only by StudioSpeechResources. + audio_prompt_type: _AudioPromptType = "" + audio_guide: StrictStr | None = None + audio_guide2: StrictStr | None = None + audio_guide3: StrictStr | None = None + audio_guide4: StrictStr | None = None + audio_guide5: StrictStr | None = None + audio_guide6: StrictStr | None = None + audio_source: Literal["", None] = None + + # Studio keeps stale image/video controls in its shared state when the + # user switches to speech. They are accepted only as explicit inactive + # sentinels so a restored speech snapshot is not silently truncated. + image_start: Literal["", None] | list[StrictStr] = None + image_end: Literal["", None] | list[StrictStr] = None + image_refs: list[StrictStr] | None = None + image_guide: Literal["", None] | list[StrictStr] = None + image_mask: Literal["", None] | list[StrictStr] = None + video_guide: Literal["", None] = None + video_mask: Literal["", None] = None + video_source: Literal["", None] = None + spatial_upsampling: Literal["", None] = "" + temporal_upsampling: Literal["", None] = "" + wangp_processor_settings: dict[str, Any] | None = None + MMAudio_setting: _Zero | None = None + MMAudio_prompt: Literal["", None] = None + MMAudio_neg_prompt: Literal["", None] = None + h3_ref_videos: list[StrictStr] | None = None + h3_ref_audios: list[StrictStr] | None = None + minimax_h3_references: list[StrictStr] | None = None + + # Private UI fields are aliases because Pydantic field names cannot begin + # with an underscore. JSON serialization uses the native aliases. + audio_sub_mode: Literal["speech"] = Field("speech", alias="_audio_sub_mode") + tts_original_prompt: _Text | None = Field(None, alias="_tts_original_prompt") + tts_speaker_name1: _Text | None = Field("", alias="_tts_speaker_name1") + tts_speaker_name2: _Text | None = Field("", alias="_tts_speaker_name2") + tts_speaker_name3: _Text | None = Field("", alias="_tts_speaker_name3") + tts_speaker_name4: _Text | None = Field("", alias="_tts_speaker_name4") + tts_speaker_name5: _Text | None = Field("", alias="_tts_speaker_name5") + tts_speaker_name6: _Text | None = Field("", alias="_tts_speaker_name6") + tts_voice_count: _VoiceCount = Field(0, alias="_tts_voice_count") + + # Model-owned timing and sampling controls. Duration zero is a real + # native sentinel for DramaBox and therefore remains valid here. + duration_seconds: _NonNegativeFinite | None = None + pause_seconds: _NonNegativeFinite | None = None + temperature: _NonNegativeFinite | None = None + top_p: Annotated[StrictFloat, Field(ge=0, le=1, allow_inf_nan=False)] | None = None + top_k: _NonNegativeInt | None = None + audio_scale: _Finite | None = None + audio_guidance_scale: _Finite | None = None + alt_scale: _Finite | None = None + guidance_phases: Annotated[StrictInt, Field(ge=0, le=16)] = 1 + flow_shift: _Finite | None = None + sample_solver: _ShortText = "" + settings_version: _NonNegativeFinite | None = None + # Speech text must reach TTS literally. Prompt enhancement is a visual + # generation feature and an active value would rewrite the speaker text. + prompt_enhancer: Literal["", None] = "" + model_mode: _ModelMode | None = None + custom_settings: StudioSpeechCustomSettings | None = None + + # Studio's audio UI currently serializes this checkbox as integer 0/1; + # older snapshots may contain a JSON boolean. Both are typed and bounded. + tts_dynaudnorm: _TtsDynaudnorm | None = None + tts_comp_threshold: _Finite | None = None + tts_comp_attack: _NonNegativeFinite | None = None + tts_comp_release: _NonNegativeFinite | None = None + tts_comp_makeup: _NonNegativeFinite | None = None + + # H3's inert sentinel appears in the shared Studio state. A true value is + # a different video/audio operation and must fail closed here. + minimax_h3_turbo_mode: StrictBool | None = None + + @field_validator( + "audio_guide", + "audio_guide2", + "audio_guide3", + "audio_guide4", + "audio_guide5", + "audio_guide6", + ) + @classmethod + def _check_audio_reference(cls, value): + if value in (None, ""): + return value + return _validate_reference(value) + + @field_validator("activated_loras") + @classmethod + def _check_lora_names(cls, values): + for value in values: + if not value.strip() or "/" in value or "\\" in value or value in {".", ".."}: + raise ValueError("activated_loras must contain exact catalog names") + return values + + @field_validator( + "image_start", + "image_end", + "image_guide", + "image_mask", + ) + @classmethod + def _check_inactive_image_slots(cls, value): + if value is None or value == "": + return value + if isinstance(value, list) and all(item == "" for item in value): + return value + raise ValueError("image references are inactive in speech mode") + + @field_validator("image_refs", "h3_ref_videos", "h3_ref_audios", "minimax_h3_references") + @classmethod + def _check_inactive_reference_lists(cls, value): + if value is None or value == [] or (isinstance(value, list) and all(item == "" for item in value)): + return value + raise ValueError("reference lists are inactive in speech mode") + + @field_validator("wangp_processor_settings") + @classmethod + def _check_inactive_processor_settings(cls, value): + if value is None or value == {}: + return value + raise ValueError("processor settings are inactive in speech mode") + + @field_validator("tts_original_prompt") + @classmethod + def _check_original_prompt(cls, value): + if value is None: + # Omission is handled by freeze_studio_speech_spec. Explicit + # JSON null is not a valid native TTS text field. + return value + return value + + @field_validator("minimax_h3_turbo_mode") + @classmethod + def _check_inactive_turbo(cls, value): + if value is True: + raise ValueError("minimax_h3_turbo_mode must be false or null for speech") + return value + + @model_validator(mode="after") + def _check_semantics(self): + _check_required_speech_texts(self) + _check_speech_prompt_metadata(self) + _check_speech_ranges(self) + return self + + +class StudioSpeechInput(_ClosedModel): + workspace: _Workspace + workspace_collection_id: _WorkspaceCollectionId | None = None + params: StudioSpeechParams + + @model_validator(mode="after") + def _check_collection_id(self): + if self.workspace_collection_id is not None and not self.workspace_collection_id.strip(): + raise ValueError("workspace_collection_id must contain a non-blank value") + return self + + +class _StudioSpeechEnvelope(_ClosedModel): + version: Literal[SCHEMA_VERSION] + operation: Literal[OPERATION] + intent_id: _IntentId + input: StudioSpeechInput + + @model_validator(mode="after") + def _check_intent(self): + if not self.intent_id.strip(): + raise ValueError("intent_id must contain a non-blank value") + return self + + +def _validation_error(exc: ValidationError) -> StudioSpeechSpecError: + details: list[dict[str, Any]] = [] + messages: list[str] = [] + for error in exc.errors(include_input=False): + location = ".".join(str(part) for part in error.get("loc", ())) or "command" + message = str(error.get("msg") or "Invalid value") + details.append({"loc": location, "message": message, "type": error.get("type")}) + messages.append(f"{location}: {message}") + return StudioSpeechSpecError( + "; ".join(messages) or "Invalid Studio speech generation command", + details=details, + ) + + +def _canonical_content(effective: dict[str, Any]) -> dict[str, Any]: + return { + "version": SCHEMA_VERSION, + "operation": OPERATION, + "input": deepcopy(effective["input"]), + } + + +def _fingerprint(content: dict[str, Any]) -> str: + encoded = json.dumps( + content, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def freeze_studio_speech_spec(command: Any) -> dict[str, Any]: + """Validate and detach one Studio speech command without side effects.""" + if type(command) is not dict: + raise StudioSpeechSpecError("Studio speech generation command must be an object") + try: + envelope = _StudioSpeechEnvelope.model_validate(command) + except ValidationError as exc: + raise _validation_error(exc) from exc + + original = deepcopy(command) + explicit_params = envelope.input.params.model_dump( + mode="json", by_alias=True, exclude_unset=True + ) + effective_params = deepcopy(explicit_params) + for key, value in STUDIO_SPEECH_DEFAULTS.items(): + effective_params.setdefault(key, deepcopy(value)) + # The original prompt is a native bookkeeping field. When Studio omits + # it, the effective projection gets the literal prompt; an explicit value + # is preserved byte-for-byte in ``original`` and by its own field. + effective_params.setdefault("_tts_original_prompt", explicit_params["prompt"]) + + effective = { + "version": SCHEMA_VERSION, + "operation": OPERATION, + "intent_id": envelope.intent_id, + "input": { + "workspace": envelope.input.workspace, + "params": effective_params, + }, + } + explicit_input = envelope.input.model_dump(mode="json", exclude_unset=True) + if "workspace_collection_id" in explicit_input: + effective["input"]["workspace_collection_id"] = explicit_input["workspace_collection_id"] + content = _canonical_content(effective) + return { + "original": original, + "effective": effective, + "fingerprint_version": FINGERPRINT_VERSION, + "fingerprint": _fingerprint(content), + } + + +def studio_speech_schema() -> dict[str, Any]: + """Return the discovery schema for the implemented speech boundary.""" + input_schema = StudioSpeechInput.model_json_schema() + envelope_schema = _StudioSpeechEnvelope.model_json_schema() + return { + "version": SCHEMA_VERSION, + "operation": OPERATION, + "intent_id": envelope_schema["properties"]["intent_id"], + "input": input_schema, + "supported_input_fields": list(SUPPORTED_INPUT_FIELDS), + "speech_model_types": sorted(SPEECH_MODEL_TYPES), + "effects": deepcopy(STUDIO_SPEECH_DEFAULTS), + "inactive": list(INACTIVE_SPEECH_FIELDS), + "excluded": list(EXCLUDED_SPEECH_FIELDS), + } + + +# Compatibility aliases used by discovery/adapters in the shared command +# slices. They point to this one contract and do not create another schema. +StudioSpeechGenerationInput = StudioSpeechInput +StudioSpeechGenerationParams = StudioSpeechParams +speech_generation_schema_v2 = studio_speech_schema + + +__all__ = [ + "EXCLUDED_SPEECH_FIELDS", + "FINGERPRINT_VERSION", + "INACTIVE_SPEECH_FIELDS", + "OPERATION", + "SCHEMA_VERSION", + "SPEECH_MODEL_TYPES", + "STUDIO_SPEECH_DEFAULTS", + "SUPPORTED_INPUT_FIELDS", + "StudioSpeechGenerationInput", + "StudioSpeechGenerationParams", + "StudioSpeechCustomSettings", + "StudioSpeechInput", + "StudioSpeechParams", + "StudioSpeechSpecError", + "freeze_studio_speech_spec", + "speech_generation_schema_v2", + "studio_speech_schema", +] diff --git a/app/services/tools_upscale_commands.py b/app/services/tools_upscale_commands.py new file mode 100644 index 000000000..bcc420eea --- /dev/null +++ b/app/services/tools_upscale_commands.py @@ -0,0 +1,228 @@ +"""Bind the typed Tools upscale contract to the shared native adapter. + +This module is deliberately an adapter only. Queue ownership, durable +receipts and worker dispatch remain in ``ImageGenerationCommands`` and the +existing launch runtime. In particular, this file does not define a second +registry or start a thread. +""" + +from __future__ import annotations + +from copy import deepcopy +import os +from typing import Any + +from services.image_generation_commands import command_error +from services.native_generation_operation import NativeGenerationOperation +from services.tools_upscale_preparation import prepare_tools_upscale +from services.tools_upscale_spec import ( + _asset_id_from_reference, + freeze_tools_upscale_spec, +) + + +def _execution_policy(runtime): + execution_mode = runtime.get("execution_mode") + validate = getattr(execution_mode, "validate_generation", None) + if not callable(validate): + return lambda _workspace: None + + def check(workspace): + try: + validate(workspace) + except Exception as error: + error_type = getattr(execution_mode, "ExecutionModeError", ValueError) + if isinstance(error, error_type): + raise command_error(409, "execution_policy", str(error)) from error + raise + + return check + + +def _resource_adapter(runtime): + from services.studio_image_resources import StudioImageResources + + workspace_dir = runtime.get("_workspace_dir") + list_workspaces = runtime.get("_list_workspaces") + if not callable(workspace_dir) or not callable(list_workspaces): + raise ValueError("Tools upscale resources are not configured") + uploads_dir = runtime.get("_uploads_dir") + if not callable(uploads_dir): + uploads_dir = lambda: os.path.join(os.getcwd(), "uploads") + wgp = runtime.get("wgp") + lora_search_dirs = getattr(wgp, "get_lora_search_dirs", None) + if not callable(lora_search_dirs): + lora_search_dirs = lambda _model_type: [] + lora_compatible = runtime.get("_lora_is_compatible_with_model") + if not callable(lora_compatible): + lora_compatible = lambda _definition, _path: True + return StudioImageResources( + workspace_dir=workspace_dir, + uploads_dir=uploads_dir, + list_workspaces=list_workspaces, + lora_search_dirs=lora_search_dirs, + lora_compatible=lora_compatible, + ) + + +def _asset_locations(runtime, asset_id): + roots_factory = runtime.get("_tool_asset_roots") + if not callable(roots_factory): + return None + from services.asset_catalog import find_asset + + asset = find_asset(roots_factory(), asset_id) + if asset is None: + return None + locations = asset.get("locations") + if not isinstance(locations, list): + raise command_error(409, "source_location_unavailable", "The source asset has no valid locations") + return locations + + +def _check_asset_scope(runtime, asset_id, source_workspace): + locations = _asset_locations(runtime, asset_id) + if locations is None: + return + if any(not isinstance(location, dict) for location in locations): + raise command_error(409, "source_location_unavailable", "The source asset has invalid locations") + matches = [item for item in locations if item.get("workspace_id") == source_workspace] + if source_workspace is not None and len(matches) != 1: + code = "ambiguous_source" if len(matches) > 1 else "source_workspace_mismatch" + raise command_error(409, code, "The source_workspace does not identify one exact asset location") + if source_workspace is None and len(locations) != 1: + code = "source_location_unavailable" if not locations else "ambiguous_source" + raise command_error(409, code, "Choose a source_workspace for this asset") + + +def _resolve_source(runtime): + resolver = runtime.get("_resolve_tool_source") or runtime.get("resolve_tool_source") + if not callable(resolver): + return None + + def resolve(params, **kwargs): + # Preparation already adapts bare/API asset references to the native + # ``asset_id`` field. Accept that prepared body as-is while also + # supporting direct adapter callers that still provide ``source``. + body = deepcopy(params) + source = body.get("source") + asset_id = body.get("asset_id") + if asset_id is None and isinstance(source, str): + asset_id = _asset_id_from_reference(source) + if asset_id: + body["asset_id"] = asset_id + body.pop("source", None) + _check_asset_scope(runtime, asset_id, body.get("source_workspace")) + expected_kinds = kwargs.get("expected_kinds") or (body.get("source_kind"),) + try: + return resolver(body, expected_kinds=expected_kinds) + except TypeError as first_error: + try: + return resolver(body) + except TypeError: + raise first_error + + return resolve + + +def _request_ready_params(params: dict[str, Any]) -> dict[str, Any]: + """Project inspected source identity into the legacy endpoint body. + + The endpoint accepts one source field. The typed adapter has already + inspected the canonical URL/asset, so it sends the exact resolved path + through the existing confined resolver until a canonical API URL can be + projected. A submitted managed asset ID is retained alongside that URL; + the native resolver checks that both identify the same basename/location. + """ + result = deepcopy(params) + source = result.pop("source", None) + asset_id = _asset_id_from_reference(source) if isinstance(source, str) else None + if asset_id: + result["asset_id"] = asset_id + path_key = "source_path" if result.get("source_kind") == "image" else "video_path" + result[path_key] = result.get(path_key) + return result + + +def _canonical_request_source(params: dict[str, Any], runtime: dict[str, Any]) -> str | None: + path_key = "source_path" if params.get("source_kind") == "image" else "video_path" + path = params.get(path_key) + workspace_dir = runtime.get("_workspace_dir") + if not path or not callable(workspace_dir): + return None + uploads_dir = runtime.get("_uploads_dir") + if not callable(uploads_dir): + uploads_dir = lambda: os.path.join(os.getcwd(), "uploads") + source_workspace = params.get("source_workspace") + if not isinstance(source_workspace, str) or not source_workspace: + return None + try: + from services.wangp_submission import wangp_media_url + + workspace_for_url = params.get("workspace") if source_workspace == "__uploads__" else source_workspace + return wangp_media_url( + path, + workspace_for_url, + uploads_dir=uploads_dir(), + workspace_dir=workspace_dir( + source_workspace if source_workspace != "__uploads__" else params.get("workspace") + ), + ) + except (OSError, TypeError, ValueError): + return None + + +def _worker(runtime): + policy = runtime["execution_mode"].policy() + return runtime["_run_generation"] if policy.simulated else runtime["_run_tool_upscale"] + + +def create_tools_upscale_operation(runtime: dict[str, Any]) -> NativeGenerationOperation: + """Create the registered ``tools.upscale`` adapter for one runtime.""" + source_resolver = _resolve_source(runtime) + + def freeze(command): + frozen = freeze_tools_upscale_spec(command) + effective = frozen["effective"]["input"] + return frozen, { + **deepcopy(effective["params"]), + "workspace": effective["workspace"], + } + + def prepare(params): + prepared, resources = prepare_tools_upscale( + params, + resources=_resource_adapter(runtime), + resolve_source=source_resolver, + execution_policy=_execution_policy(runtime), + processor_capabilities=runtime.get("_tools_processor_capabilities"), + validate_processors=runtime.get("_validate_tools_processors"), + processor_settings=runtime.get("_validated_tools_processor_settings"), + processor_parameters=runtime.get("_tools_processor_parameters"), + probe_video=runtime.get("_probe_tool_video"), + ) + ready = _request_ready_params(prepared) + canonical_source = _canonical_request_source(prepared, runtime) + if canonical_source: + path_key = "source_path" if prepared.get("source_kind") == "image" else "video_path" + ready.pop(path_key, None) + ready["source"] = canonical_source + return ready, resources + + from routers.tools_upscale_commands import tools_upscale_command_catalog + + return NativeGenerationOperation( + freeze=freeze, + prepare=prepare, + catalog=tools_upscale_command_catalog(), + prepare_request=runtime["tools_upscale"], + worker=_worker(runtime), + use_generation_defaults=False, + ) + + +# Alias kept for discovery code that calls adapters by their operation name. +create_tools_upscale_adapter = create_tools_upscale_operation + + +__all__ = ["create_tools_upscale_adapter", "create_tools_upscale_operation"] diff --git a/app/services/tools_upscale_preparation.py b/app/services/tools_upscale_preparation.py new file mode 100644 index 000000000..daf34d6cb --- /dev/null +++ b/app/services/tools_upscale_preparation.py @@ -0,0 +1,412 @@ +"""Provider-free source and processor preparation for ``tools.upscale``. + +The preparation boundary runs after the typed command has been frozen and +before the shared admission callback is invoked. It uses the existing Tools +source resolver and WangGP processor validators, then returns a detached +native worker snapshot plus portable resource identities. It never starts a +worker, calls an endpoint, downloads a model or writes an output. +""" + +from __future__ import annotations + +from copy import deepcopy +from collections.abc import Callable, Mapping +import os +from pathlib import Path +from typing import Any + +from fastapi import HTTPException +from pydantic import ValidationError + +from services.image_generation_commands import command_error +from services.studio_image_resources import file_identity +from services.tools_upscale import TOOL_SOURCE_EXTENSIONS +from services.tools_upscale_spec import ToolsUpscaleParams + + +def _processor_defaults() -> tuple[Callable, Callable, Callable]: + from shared.wangp1272 import processors + + return processors.capabilities, processors.validate_selection, processors.validated_settings + + +def _validated_params(params: Any) -> tuple[dict[str, Any], str]: + if type(params) is not dict: + raise command_error(422, "invalid_tools_upscale_input", "Tools upscale parameters must be an object") + workspace = params.get("workspace") + if not isinstance(workspace, str) or not workspace.strip(): + raise command_error(422, "invalid_workspace", "Use an explicit output workspace") + try: + parsed = ToolsUpscaleParams.model_validate( + {key: value for key, value in params.items() if key != "workspace"} + ) + except ValidationError as error: + details = "; ".join( + f"{'.'.join(str(part) for part in item.get('loc', ())) or 'input.params'}: " + f"{item.get('msg') or 'Invalid value'}" + for item in error.errors(include_input=False) + ) + raise command_error(422, "invalid_tools_upscale_input", details) from error + working = parsed.model_dump(mode="json") + # Only submitted settings belong to this processor. Nested model defaults + # include an inactive H3 reference list which is not a Lanczos parameter. + settings = parsed.wangp_processor_settings + working["wangp_processor_settings"] = settings.model_dump(mode="json", exclude_unset=True) if settings else {} + working["workspace"] = workspace + return working, workspace + + +def _call_source_resolver(resolver: Callable, params: dict[str, Any], kind: str): + request = { + "source": params["source"], + "source_kind": kind, + "workspace": params["workspace"], + } + # The legacy runtime resolver accepts asset_id separately. Keep the + # typed contract's single canonical ``source`` field while adapting only + # this in-process call; the frozen receipt still contains the source ID. + source = str(params["source"]) + if source.startswith(("asset_", "asset:", "asset-")): + request["asset_id"] = source + request.pop("source", None) + elif source.startswith("/api/v1/assets/"): + from services.tools_upscale_spec import _asset_id_from_reference + + request["asset_id"] = _asset_id_from_reference(source) + request.pop("source", None) + if params.get("source_workspace") is not None: + request["source_workspace"] = params["source_workspace"] + try: + return resolver(request, expected_kinds=(kind,)) + except TypeError as first_error: + try: + return resolver(request) + except TypeError: + raise first_error + + +def _fallback_source(params: dict[str, Any], resources: Any): + if str(params.get("source_kind")) != "image" or not hasattr(resources, "_media"): + raise ValueError("A canonical Tools source resolver is required for this media kind") + path, source_workspace = resources._media(params["source"]) + from services.tools_upscale_spec import _asset_id_from_reference + + return { + "path": path, + "filename": os.path.basename(os.fspath(path)), + "source_workspace": source_workspace, + "source_kind": "image", + "asset_id": _asset_id_from_reference(str(params["source"])), + } + + +def _source_fields(result: Any) -> dict[str, Any]: + if isinstance(result, Mapping): + values = { + "path": result.get("path") or result.get("source_path") or result.get("resolved"), + "filename": result.get("filename") or result.get("source_filename"), + "source_workspace": result.get("source_workspace") or result.get("workspace"), + "source_kind": result.get("source_kind") or result.get("kind"), + "asset_id": result.get("asset_id") or result.get("source_asset_id"), + "output_workspace": result.get("output_workspace"), + "output_dir": result.get("output_dir"), + } + elif isinstance(result, (tuple, list)) and len(result) >= 4: + values = { + "path": result[0], "filename": result[1], "source_workspace": result[2], + "source_kind": result[3], "asset_id": result[4] if len(result) > 4 else None, + "output_workspace": result[5] if len(result) > 5 else None, + "output_dir": result[6] if len(result) > 6 else None, + } + else: + raise ValueError("The Tools source resolver returned an invalid result") + return values + + +def _normalise_source(result: Any, expected_kind: str) -> dict[str, Any]: + values = _source_fields(result) + path = values["path"] + if not isinstance(path, (str, os.PathLike)) or not os.fspath(path): + raise ValueError("The selected Tools source is unavailable") + path = os.fspath(path) + filename = values["filename"] or os.path.basename(path) + if not isinstance(filename, str) or os.path.basename(filename) != filename: + raise ValueError("The selected source filename is invalid") + source_workspace = values["source_workspace"] + if not isinstance(source_workspace, str) or not source_workspace.strip(): + raise ValueError("The selected source has no source workspace") + source_kind = values["source_kind"] or expected_kind + if source_kind != expected_kind: + raise ValueError("Source kind does not match the typed command") + return {**values, "path": path, "filename": filename, "source_kind": source_kind} + + +def _confined_to_resource_root(path: str, source_workspace: str, resources: Any) -> None: + root_factory = getattr(resources, "uploads_dir", None) if source_workspace == "__uploads__" else getattr(resources, "workspace_dir", None) + if not callable(root_factory): + return + try: + root = Path(root_factory() if source_workspace == "__uploads__" else root_factory(source_workspace)).resolve() + resolved = Path(path).resolve() + except (OSError, TypeError, ValueError) as error: + raise ValueError("The selected source location is unavailable") from error + if not resolved.is_relative_to(root) or resolved == root: + raise ValueError("The selected source is outside its declared workspace") + + +def _check_source_path(source: dict[str, Any], resources: Any) -> Path: + path = Path(source["path"]) + if not path.is_file(): + raise ValueError("The selected source is unavailable") + _confined_to_resource_root(os.fspath(path), source["source_workspace"], resources) + suffix = path.suffix.casefold() + if suffix not in TOOL_SOURCE_EXTENSIONS[source["source_kind"]]: + raise ValueError("Source kind does not match the file format") + return path + + +def _inspect_image(path: Path) -> dict[str, Any]: + from PIL import Image + + try: + with Image.open(path) as picture: + picture.verify() + with Image.open(path) as picture: + return {"format": picture.format, "width": picture.width, "height": picture.height} + except (OSError, SyntaxError, ValueError) as error: + raise ValueError("The selected image could not be decoded") from error + + +def _inspect_video(path: Path, probe_video: Callable | None) -> dict[str, Any]: + probe = probe_video + if probe is None: + from services.video_editor import probe_media + + probe = probe_media + try: + metadata = probe(os.fspath(path)) + except Exception as error: + raise ValueError("The selected video could not be inspected") from error + if not isinstance(metadata, Mapping): + raise ValueError("The video probe returned invalid metadata") + try: + duration = float(metadata.get("duration") or 0) + width = int(metadata.get("width") or 0) + height = int(metadata.get("height") or 0) + except (TypeError, ValueError) as error: + raise ValueError("The video probe returned invalid metadata") from error + import math + + if not math.isfinite(duration) or duration <= 0 or width <= 0 or height <= 0: + raise ValueError("The selected video has no readable duration or dimensions") + return deepcopy(dict(metadata)) + + +def _source_resource(source: dict[str, Any], path: Path, resources: Any, probe_video: Callable | None) -> dict[str, Any]: + identity = file_identity(path) + metadata = (_inspect_image(path) if source["source_kind"] == "image" + else _inspect_video(path, probe_video)) + return { + "role": "source", + "index": 0, + "url": source.get("url"), + "workspace": source["source_workspace"], + "kind": source["source_kind"], + "filename": source["filename"], + "asset_id": source.get("asset_id"), + "media": metadata, + **identity, + } + + +def _resolve_reference(resources: Any, value: str): + resolver = getattr(resources, "resolve_reference", None) + if callable(resolver): + try: + return resolver(value, media_kind="image") + except TypeError as first_error: + try: + return resolver(value) + except TypeError: + raise first_error + media = getattr(resources, "_media", None) + if callable(media): + return media(value) + raise ValueError("Processor references require the existing media resolver") + + +def _processor_reference_resources(settings: dict[str, Any], resources: Any) -> tuple[dict[str, Any], list[dict[str, Any]]]: + values = settings.get("spatial_upsampler_reference_images") or [] + if not values: + return settings, [] + prepared: list[str] = [] + identities: list[dict[str, Any]] = [] + for index, value in enumerate(values): + try: + path, workspace = _resolve_reference(resources, value) + except (OSError, TypeError, ValueError) as error: + raise ValueError(f"Processor reference {index} is unavailable") from error + path = Path(path) + if not path.is_file(): + raise ValueError(f"Processor reference {index} is unavailable") + _confined_to_resource_root(os.fspath(path), workspace, resources) + identities.append({ + "role": "processor_reference", "index": index, "url": value, + "workspace": workspace, **file_identity(path), "media": _inspect_image(path), + }) + prepared.append(os.fspath(path)) + result = deepcopy(settings) + result["spatial_upsampler_reference_images"] = prepared + return result, identities + + +def _declared_parameters(method: str, resolver: Callable | None = None) -> dict[str, dict[str, Any]]: + if resolver is not None: + try: + values = resolver(method) + except (ImportError, AttributeError, RuntimeError): + values = [] + else: + try: + from postprocessing.spatial_upsamplers import method_parameters + + values = method_parameters(method) + except (ImportError, AttributeError, RuntimeError): + values = [] + if isinstance(values, Mapping): + values = values.values() + if not isinstance(values, (list, tuple)): + values = [] + return { + str(item["name"]): item + for item in values + if isinstance(item, Mapping) and isinstance(item.get("name"), str) + } + + +def _validate_processor_settings(method: str, settings: dict[str, Any], validator: Callable, + parameter_resolver: Callable | None = None) -> dict[str, Any]: + submitted = {key: value for key, value in settings.items() if value is not None} + resolved = validator(method, submitted) + if not isinstance(resolved, dict): + raise ValueError("Processor settings validator returned an invalid object") + declared = _declared_parameters(method, parameter_resolver) + unsupported = set(submitted) - set(resolved) + unsupported -= { + key for key in unsupported + if declared.get(key, {}).get("type") == "array" + } + if unsupported: + raise ValueError("Processor settings contain values not declared by the selected processor") + result = deepcopy(resolved) + for key in set(submitted) - set(resolved): + if declared.get(key, {}).get("type") != "array": + continue + result[key] = deepcopy(submitted[key]) + return result + + +def _validate_processor(method: str, source_kind: str, capabilities: Callable | None, + selection_validator: Callable, settings_validator: Callable, + settings: dict[str, Any], parameter_resolver: Callable | None = None) -> dict[str, Any]: + temporal = method.startswith(("rife", "dlssg")) + spatial_value, temporal_value = ("", method) if temporal else (method, "") + error = selection_validator(spatial_value, temporal_value, source_kind == "image") + if error: + raise ValueError(str(error)) + if callable(capabilities): + records = capabilities() + if isinstance(records, (list, tuple)): + selected = next((item for item in records if isinstance(item, Mapping) and item.get("value") == method), None) + if selected is not None: + if selected.get("enabled") is False: + raise ValueError("The selected processor is disabled") + if source_kind not in selected.get("media", (source_kind,)): + raise ValueError("The selected processor does not support this source kind") + return _validate_processor_settings(method, settings, settings_validator, parameter_resolver) + + +def _native_snapshot(working: dict[str, Any], source: dict[str, Any], path: Path, + settings: dict[str, Any]) -> dict[str, Any]: + source_ref = { + "id": source.get("asset_id"), "kind": source["source_kind"], + "uri": source["filename"], "role": "source", + } + result = deepcopy(working) + result.update({ + "source_path": os.fspath(path) if source["source_kind"] == "image" else None, + "video_path": os.fspath(path) if source["source_kind"] == "video" else None, + "source_filename": source["filename"], + "source_workspace": source["source_workspace"], + "source_asset_id": source.get("asset_id"), + "model_type": "post_processing", + "generation_mode": source["source_kind"], + "provider": "local", + "capability": "tools.upscale", + "inputs": [source_ref], + "parents": [source_ref], + "transformations": [{ + "type": "upscale", "method": working["method"], + }], + "wangp_processor_settings": deepcopy(settings), + }) + return result + + +def prepare_tools_upscale( + params: dict[str, Any], *, resources: Any, resolve_source: Callable | None = None, + execution_policy: Callable | None = None, processor_capabilities: Callable | None = None, + validate_processors: Callable | None = None, processor_settings: Callable | None = None, + processor_parameters: Callable | None = None, probe_video: Callable | None = None, +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """Resolve and inspect one typed Tools source before shared admission.""" + working, workspace = _validated_params(params) + try: + if execution_policy is not None: + execution_policy(workspace) + defaults = None + if processor_capabilities is None or validate_processors is None or processor_settings is None: + defaults = _processor_defaults() + capabilities = processor_capabilities or defaults[0] + selection_validator = validate_processors or defaults[1] + settings_validator = processor_settings or defaults[2] + resolver = resolve_source or getattr(resources, "resolve_source", None) + kind = working["source_kind"] + result = (_call_source_resolver(resolver, working, kind) if callable(resolver) + else _fallback_source(working, resources)) + source = _normalise_source(result, kind) + requested_workspace = working.get("source_workspace") + if requested_workspace is not None and source["source_workspace"] != requested_workspace: + raise ValueError("The selected source does not match source_workspace") + source["url"] = working["source"] + path = _check_source_path(source, resources) + identities = [_source_resource(source, path, resources, probe_video)] + # Resolve processor reference media first. The native validator then + # sees the exact confined paths it will receive, while the frozen + # receipt retains submitted URLs and independent identities. + settings, processor_refs = _processor_reference_resources( + working["wangp_processor_settings"], resources + ) + settings = _validate_processor( + working["method"], kind, capabilities, selection_validator, + settings_validator, settings, processor_parameters, + ) + identities.extend(processor_refs) + return _native_snapshot(working, source, path, settings), deepcopy(identities) + except HTTPException: + raise + except (OSError, ValueError) as error: + raise command_error(422, "invalid_tools_upscale_input", str(error)) from error + + +# Descriptive aliases used by adapter discovery and tests in the other command +# slices. They all call this one preparation boundary. +prepare_tools_upscale_command = prepare_tools_upscale +prepare_tools_upscale_inputs = prepare_tools_upscale + + +__all__ = [ + "prepare_tools_upscale", + "prepare_tools_upscale_command", + "prepare_tools_upscale_inputs", +] diff --git a/app/services/tools_upscale_spec.py b/app/services/tools_upscale_spec.py new file mode 100644 index 000000000..796f7b93d --- /dev/null +++ b/app/services/tools_upscale_spec.py @@ -0,0 +1,357 @@ +"""Closed, provider-free command contract for ``tools.upscale``. + +The Tools worker already has a native implementation for spatial and temporal +upscaling. This module only freezes the small transport boundary that lets +Wizard, Studio and an external agent submit that implementation through the +shared admission path. It does not resolve a source, inspect a file, load a +processor or enqueue a job. + +The envelope is:: + + { + "version": 2, + "operation": "tools.upscale", + "intent_id": "...", + "input": { + "workspace": "...", + "workspace_collection_id": "...", + "params": { + "source": "asset_... or /api/v1/...", + "source_workspace": "source or __uploads__", + "source_kind": "image" or "video", + "method": "lanczos2", + "seed": -1, + "wangp_processor_settings": { ... } + } + } + } + +``original`` is a detached copy of the submitted envelope. ``effective`` +adds only the deterministic seed/settings defaults and retains the source +spelling exactly. The fingerprint covers operation, workspace and effective +parameters, excluding the transport intent and caller metadata. +""" + +from __future__ import annotations + +from copy import deepcopy +import hashlib +import json +from typing import Annotated, Any, Literal +from urllib.parse import parse_qs, unquote, urlsplit + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StringConstraints, + StrictInt, + StrictStr, + ValidationError, + field_validator, + model_validator, +) + +from services.image_generation_spec import ImageGenerationSpecError +from services.studio_image_spec import WangpProcessorSettings, _validate_reference +from services.tools_upscale import TOOL_UPSCALE_METHODS + + +SCHEMA_VERSION = 2 +FINGERPRINT_VERSION = 2 +OPERATION = "tools.upscale" + +_MAX_ID_LENGTH = 240 +_MAX_INTENT_LENGTH = 160 +_MAX_REFERENCE_LENGTH = 8192 +_SEED_MIN = -(2**63) +_SEED_MAX = 2**63 - 1 + +TOOLS_UPSCALE_DEFAULTS: dict[str, Any] = { + "seed": -1, + "wangp_processor_settings": {}, +} + +SUPPORTED_SOURCE_KINDS = ("image", "video") +SUPPORTED_METHODS = tuple(sorted(TOOL_UPSCALE_METHODS)) + +# These processors are explicitly temporal or require a video in the existing +# Tools implementation. The definitive availability/configuration decision +# remains the WangGP processor validator in the preparation boundary. +_VIDEO_ONLY_METHODS = frozenset( + method + for method in TOOL_UPSCALE_METHODS + if method == "h3facerefine" or method.startswith(("rife", "dlssg")) +) + + +class ToolsUpscaleSpecError(ImageGenerationSpecError): + """Validation error for the closed Tools upscale envelope.""" + + +class _ClosedModel(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + +_Workspace = Annotated[ + StrictStr, + StringConstraints( + min_length=1, + max_length=_MAX_ID_LENGTH, + pattern=r"^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$", + ), +] +_WorkspaceCollectionId = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=200), +] +_IntentId = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=_MAX_INTENT_LENGTH), +] +_Source = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=_MAX_REFERENCE_LENGTH), +] +_SourceWorkspace = Annotated[ + StrictStr, + StringConstraints( + min_length=1, + max_length=160, + pattern=r"^(?:__uploads__|default|[A-Za-z0-9][A-Za-z0-9_-]*)$", + ), +] +_Method = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=80), +] +_Seed = Annotated[StrictInt, Field(ge=_SEED_MIN, le=_SEED_MAX)] + + +class ToolsUpscaleParams(_ClosedModel): + """Typed native input for one image or video upscale.""" + + source: _Source + # An asset ID can exist in more than one explicit media root. This scope + # selects that exact source location; it is kept in the native snapshot. + source_workspace: _SourceWorkspace | None = None + source_kind: Literal["image", "video"] + method: _Method + seed: _Seed = -1 + wangp_processor_settings: WangpProcessorSettings | None = None + + @field_validator("source") + @classmethod + def _canonical_source(cls, value: str) -> str: + # The shared image reference validator accepts exact asset IDs and the + # same local API URL families used by Studio resources. It rejects + # absolute host paths, remote URLs, fragments, traversal and duplicate + # source-workspace query values without changing caller spelling. + try: + return _validate_reference(value) + except ValueError as error: + raise ValueError(str(error)) from error + + @field_validator("method") + @classmethod + def _known_method(cls, value: str) -> str: + if value not in TOOL_UPSCALE_METHODS: + raise ValueError("method must be one of the installed Tools upscale methods") + return value + + @model_validator(mode="after") + def _check_source_workspace(self): + parsed = urlsplit(self.source) + expected = None + if parsed.path.startswith("/api/v1/uploads/"): + expected = "__uploads__" + elif parsed.path.startswith("/api/v1/file/"): + values = parse_qs(parsed.query, keep_blank_values=True).get("workspace", []) + if len(values) == 1: + expected = values[0] + if expected and self.source_workspace not in (None, expected): + raise ValueError("source_workspace must match the source URL workspace") + return self + + @model_validator(mode="after") + def _check_media_method(self): + if self.source_kind == "image" and self.method in _VIDEO_ONLY_METHODS: + raise ValueError("the selected processor requires a video source") + return self + + +class ToolsUpscaleInput(_ClosedModel): + workspace: _Workspace + # Collection identity is transport provenance. It is retained in the + # receipt/fingerprint but is not forwarded as a native processor option. + workspace_collection_id: _WorkspaceCollectionId | None = None + params: ToolsUpscaleParams + + @model_validator(mode="after") + def _check_collection_id(self): + if self.workspace_collection_id is not None and not self.workspace_collection_id.strip(): + raise ValueError("workspace_collection_id must contain a non-blank value") + return self + + @field_validator("workspace") + @classmethod + def _nonblank_workspace(cls, value: str) -> str: + if not value.strip(): + raise ValueError("input.workspace must be a non-blank output workspace") + return value + + +class _ToolsUpscaleEnvelope(_ClosedModel): + version: Literal[SCHEMA_VERSION] + operation: Literal[OPERATION] + intent_id: _IntentId + input: ToolsUpscaleInput + + +def _validation_error(exc: ValidationError) -> ToolsUpscaleSpecError: + details: list[dict[str, Any]] = [] + messages: list[str] = [] + for error in exc.errors(include_input=False): + location = ".".join(str(part) for part in error.get("loc", ())) or "command" + message = str(error.get("msg") or "Invalid value") + details.append({"loc": location, "message": message, "type": error.get("type")}) + messages.append(f"{location}: {message}") + return ToolsUpscaleSpecError( + "; ".join(messages) or "Invalid Tools upscale command", details=details + ) + + +def _canonical_content(effective: dict[str, Any]) -> dict[str, Any]: + return { + "version": SCHEMA_VERSION, + "operation": OPERATION, + "input": deepcopy(effective["input"]), + } + + +def _fingerprint(content: dict[str, Any]) -> str: + encoded = json.dumps( + content, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _asset_id_from_reference(value: str) -> str | None: + """Return the exact asset ID represented by a source reference.""" + if value.startswith(("asset_", "asset:", "asset-")): + return value + parsed = urlsplit(value) + if parsed.path.startswith("/api/v1/assets/") and not parsed.query: + return unquote(parsed.path[len("/api/v1/assets/"):]) + return None + + +def freeze_tools_upscale_spec(command: Any) -> dict[str, Any]: + """Validate and detach one Tools upscale command without side effects.""" + if type(command) is not dict: + raise ToolsUpscaleSpecError("Tools upscale command must be an object") + try: + envelope = _ToolsUpscaleEnvelope.model_validate(command) + except ValidationError as exc: + raise _validation_error(exc) from exc + + if not envelope.intent_id.strip(): + raise ToolsUpscaleSpecError("intent_id must contain a non-blank value") + + original = deepcopy(command) + explicit_params = envelope.input.params.model_dump(mode="json", exclude_unset=True) + effective_params = deepcopy(explicit_params) + for key, value in TOOLS_UPSCALE_DEFAULTS.items(): + effective_params.setdefault(key, deepcopy(value)) + effective = { + "version": SCHEMA_VERSION, + "operation": OPERATION, + "intent_id": envelope.intent_id, + "input": { + "workspace": envelope.input.workspace, + "params": effective_params, + }, + } + # Preserve omission versus an explicit null, matching the image and + # speech command contracts. The collection id participates in the + # content fingerprint because it identifies the intended collection. + explicit_input = envelope.input.model_dump(mode="json", exclude_unset=True) + if "workspace_collection_id" in explicit_input: + effective["input"]["workspace_collection_id"] = explicit_input["workspace_collection_id"] + return { + "original": original, + "effective": effective, + "fingerprint_version": FINGERPRINT_VERSION, + "fingerprint": _fingerprint(_canonical_content(effective)), + } + + +def tools_upscale_schema() -> dict[str, Any]: + """Return the executable v2 schema used by HTTP, MCP and discovery.""" + input_schema = ToolsUpscaleInput.model_json_schema() + return { + "version": SCHEMA_VERSION, + "operation": OPERATION, + "intent_id": { + "type": "string", + "minLength": 1, + "maxLength": _MAX_INTENT_LENGTH, + }, + "input": input_schema, + "supported_input_fields": [ + "workspace", + "workspace_collection_id", + "source", + "source_workspace", + "source_kind", + "method", + "seed", + "wangp_processor_settings", + ], + "source_kinds": list(SUPPORTED_SOURCE_KINDS), + "methods": list(SUPPORTED_METHODS), + "effects": deepcopy(TOOLS_UPSCALE_DEFAULTS), + "excluded": [ + "actor", + "client", + "permission", + "provenance", + "workspace inside input.params", + "filesystem paths", + "remote URLs", + "free-form processor settings", + "generation.image", + "generation.speech", + ], + } + + +# Compatibility names used by the other shared-command slices. They all +# point at this one contract and do not create a second schema version. +freeze_tools_upscale_command = freeze_tools_upscale_spec +tools_upscale_schema_v2 = tools_upscale_schema +ToolsUpscaleGenerationInput = ToolsUpscaleInput +ToolsUpscaleGenerationParams = ToolsUpscaleParams + + +__all__ = [ + "FINGERPRINT_VERSION", + "OPERATION", + "SCHEMA_VERSION", + "SUPPORTED_METHODS", + "SUPPORTED_SOURCE_KINDS", + "TOOLS_UPSCALE_DEFAULTS", + "ToolsUpscaleGenerationInput", + "ToolsUpscaleGenerationParams", + "ToolsUpscaleInput", + "ToolsUpscaleParams", + "ToolsUpscaleSpecError", + "freeze_tools_upscale_command", + "freeze_tools_upscale_spec", + "tools_upscale_schema", + "tools_upscale_schema_v2", +] diff --git a/app/services/wangp_submission.py b/app/services/wangp_submission.py index 895dae49a..eea5d3c00 100644 --- a/app/services/wangp_submission.py +++ b/app/services/wangp_submission.py @@ -20,8 +20,7 @@ async def json(self): return deepcopy(self.payload) -def prepare_generation_inputs(body, model_def, workspace, *, uploads_dir, workspace_dir, prepared_images=False): - """Validate processor options and resolve new-family media before admission.""" +def _prepare_canonical_image_references(body, model_def, workspace, *, uploads_dir, workspace_dir): canonical_refs = body.pop('canonical_image_refs', False) if canonical_refs: if canonical_refs is not True or not model_def.get('image_outputs') or body.get('image_mode') != 1: @@ -31,23 +30,39 @@ def prepare_generation_inputs(body, model_def, workspace, *, uploads_dir, worksp raise ValueError('Canonical image references must be a non-empty ordered list') body['image_refs'] = [resolve_wangp_media(value, workspace, uploads_dir=uploads_dir, workspace_dir=workspace_dir) for value in references] + + +def _prepare_native_processors(body): if body.get('spatial_upsampling') or body.get('temporal_upsampling') or body.get('wangp_processor_settings'): from shared.wangp1272.processors import validate_selection, validated_settings error = validate_selection(body.get('spatial_upsampling', ''), body.get('temporal_upsampling', ''), body.get('image_mode') == 1) if error: raise ValueError(error) body['wangp_processor_settings'] = validated_settings(body.get('spatial_upsampling', ''), body.get('wangp_processor_settings')) + + +def prepare_generation_inputs(body, model_def, workspace, *, uploads_dir, workspace_dir, + prepared_images=False, prepared_speech=False): + """Validate processor options and resolve new-family media before admission.""" + _prepare_canonical_image_references(body, model_def, workspace, + uploads_dir=uploads_dir, workspace_dir=workspace_dir) + _prepare_native_processors(body) if not model_def.get('wangp_1272'): return if prepared_images and (not model_def.get('image_outputs') or body.get('image_mode') != 1): raise ValueError('Prepared image inputs require an image generation request') + if prepared_speech and (not model_def.get('audio_only') or body.get('generation_mode') != 'audio'): + raise ValueError('Prepared speech inputs require an audio generation request') def resolve(value): return resolve_wangp_media(value, workspace, uploads_dir=uploads_dir, workspace_dir=workspace_dir) - for field in ('video_guide', 'video_guide2', 'video_mask', 'audio_guide', 'audio_guide2', 'image_start', 'image_end', 'image_refs'): + audio_fields = ('audio_guide', 'audio_guide2', 'audio_guide3', 'audio_guide4', 'audio_guide5', 'audio_guide6') + for field in ('video_guide', 'video_guide2', 'video_mask', *audio_fields, 'image_start', 'image_end', 'image_refs'): if prepared_images and field in ('image_start', 'image_end', 'image_refs'): # Only the in-process Studio command adapter supplies this flag. # Those exact paths were resolved against each source workspace. continue + if prepared_speech and field in audio_fields: + continue values = body.get(field) if values: body[field] = [resolve(value) for value in values] if isinstance(values, list) else resolve(values) diff --git a/docs/development/LABS_WIZARD_ACTION_MATRIX.md b/docs/development/LABS_WIZARD_ACTION_MATRIX.md index 7a58995e1..ecf42dfea 100644 --- a/docs/development/LABS_WIZARD_ACTION_MATRIX.md +++ b/docs/development/LABS_WIZARD_ACTION_MATRIX.md @@ -1174,6 +1174,7 @@ These IDs are in `AGENT_ACTION_TYPES`. They are listed so L0 can prove every Wiz | `other.attach_studio_references` | `attach_studio_references` | `attachReferences` | `ui/src/features/agent/studioCapabilities.ts` | `studio.attachReferences` | | `other.configure_studio_loras` | `configure_studio_loras` | `configureLoras` | `ui/src/features/agent/studioCapabilities.ts` | `studio.configureLoras` | | `other.remove_background` | `remove_background` | `removeBackground` | `ui/src/features/agent/toolCapabilities.ts` | `tools.removeBackground` | +| `other.upscale` | `upscale` | `upscale` | `ui/src/features/agent/toolCapabilities.ts` | `tools.upscale` | | `other.open_3d_scene` | `open_3d_scene` | `open` | `ui/src/features/agent/applicationAdapters.ts` | `video3d.open` | | `other.save_3d_scene` | `save_3d_scene` | `run` | `ui/src/features/agent/applicationAdapters.ts` | `video3d.run` | | `other.export_3d_scene` | `export_3d_scene` | `run` | `ui/src/features/agent/applicationAdapters.ts` | `video3d.run` | diff --git a/docs/development/MUSIC_COMMANDS.md b/docs/development/MUSIC_COMMANDS.md new file mode 100644 index 000000000..1422f7027 --- /dev/null +++ b/docs/development/MUSIC_COMMANDS.md @@ -0,0 +1,132 @@ +# Studio music commands + +Status: local, provider-free command boundary for `generation.music`, schema +version 2. This slice freezes and preflights one Studio song request; the +runtime adapter supplies the existing generation queue and worker. + +## Envelope + +```json +{ + "version": 2, + "operation": "generation.music", + "intent_id": "song-intent-1", + "input": { + "workspace": "my-workspace", + "workspace_collection_id": "collection-id", + "params": { + "model_type": "ace_step_v1_5_xl_sft_lm_4b", + "prompt": "[Verse]\nLiteral lyrics", + "alt_prompt": "Warm acoustic pop with brushed drums", + "_music_description": "Optional UI description, retained literally", + "_music_instrumental": false, + "duration_seconds": 20, + "seed": 42, + "num_inference_steps": 8, + "guidance_scale": 1.0, + "generation_mode": "audio", + "_audio_sub_mode": "music", + "image_mode": 0, + "video_length": 0 + } + } +} +``` + +`prompt` is the native lyrics field and `alt_prompt` is the native Music +Caption. Both values are retained byte-for-byte, including surrounding spaces +and newlines. `_music_description` is UI metadata and is never used as a +fallback for `alt_prompt`. `_music_instrumental` is retained as metadata and +does not rewrite the lyrics; the caller must send the native +`[Instrumental]` literal when that is the intended prompt. When supplied, +`lyrics_language` is retained literally and passed to the existing language +guard; it does not translate or rewrite the lyrics. + +`freeze_studio_music_spec` returns detached `original` and `effective` +snapshots. The latter adds only deterministic Studio selectors and inactive +sentinels. The fingerprint covers version, operation, workspace, optional +collection ID and effective native parameters, while excluding `intent_id`. +The module does not call `freeze_music_spec`, which strips Story text and has +different remote/local semantics. + +## Supported native surface + +The first vertical registers exactly the installed local IDs +`ace_step_v1_5_xl_sft_lm_4b` and `minimax_music3`. Remote MiniMax, community +ports, SFX/MMAudio, speech, video, image, avatar and model3d controls are +outside this operation and are rejected. The native mode is always +`generation_mode="audio"` and `_audio_sub_mode="music"`; `image_mode=0`, +`video_length=0`, `multi_prompts_gen_type=2`, `repeat_generation=1` and +`batch_size=1` are exact sentinels. Active negative prompts and prompt +enhancement are rejected because both local handlers declare no negative +prompt support and enhancement would rewrite literal lyrics. + +The command accepts the model-owned timing and sampling fields +`duration_seconds`, `num_inference_steps`, `guidance_scale`, `seed`, +`guidance_phases`, `temperature`, `top_p`, `top_k`, `audio_scale`, +`alt_guidance_scale`, `sample_solver`, `model_mode`, `settings_version` and +closed ACE custom settings (`bpm`, `keyscale`, `timesignature`, `language`). +The preparation layer fills omitted duration/steps/guidance/phases from the +selected native model definition or the documented handler defaults. Studio +uses the selected native handler's declared `duration_slider` minimum and +maximum; the Story-only `music_model_contract` minimum of 20 seconds is not +applied to Studio requests. The catalog maximum remains a safety ceiling. A +value outside those native controls is rejected before resource preparation; +it is never silently clipped. + +ACE-Step's declared source selectors are `""`, `"A"`, `"B"` and `"AB"`. +`audio_guide` and `audio_guide2` are required exactly when their selector is +active. `audio_guide3` through `audio_guide6` are inactive for this music +handler. MiniMax-Music3 rejects every reference. References must be asset IDs +or canonical local API URLs and are resolved by `StudioSpeechResources`; host +URLs, traversal and absolute host paths do not cross this boundary. + +LoRAs are accepted only when the selected model definition explicitly exposes +`enabled_audio_lora`; names, multipliers, model roots and file identities are +checked by the existing resource resolver. MiniMax-Music3 currently does not +advertise that capability. No model download, fallback or remote LLM is +performed by this command. + +## Shared Studio metadata + +The browser's common audio path currently serializes `_tts_original_prompt`, +`_tts_speaker_name1` through `_tts_speaker_name6` and `_tts_voice_count` even +for music. `_tts_original_prompt` is retained as bookkeeping (defaulting to +the literal lyrics only when omitted). Speaker names must be empty/null and +the voice count must be zero. Active voices fail closed rather than being +treated as music references. The UI adapter must project these known inactive +sentinels and must not send voice references or voice counts as a music +command. + +## Runtime boundary and limits + +`app/services/studio_music_preparation.py` calls the supplied execution policy +before model lookup, verifies the exact model definition and installed flag, +then prepares canonical resources. It returns detached native params and +resource identities. The parent runtime connects this result to the existing +`NativeGenerationOperation`, `TaskRegistry` and generation FIFO. This slice +does not create a music-specific reservation store, scheduler or worker and +does not cover Story's remote `MusicSubmissionStore` path. + +The native facade remains responsible for final handler validation and actual +generation. Provider-free tests prove the contract, model capability guards, +reference/LoRA boundaries and input immutability; they do not prove GPU +inference or audio quality. + +## Wizard authored fields + +Studio and the Wizard capability use the same audio action parser. For an +explicit Studio Music execution request, named `prompt` (or `Lyrics/prompt`), +`alt_prompt` and `music_description` sections in the user message take precedence +over an LLM rewrite. A multiline section must end at the next named section; +inline fields end at the newline. Duplicate labels are ambiguous and are not +reconciled. This preserves spaces and line breaks for these bounded fields; it +is not a guarantee of literal extraction from arbitrary prose. + +### Optional ACE-Step caption + +ACE-Step accepts an empty or omitted `alt_prompt`; lyrics and `[Instrumental]` +requests do not require a style description. MiniMax-Music3 still requires a +nonblank caption. The UI builder and server freeze validate that distinction +before resource inspection or admission, preserving the original text and the +difference between an omitted field and an explicitly empty field. diff --git a/docs/development/SFX_COMMANDS.md b/docs/development/SFX_COMMANDS.md new file mode 100644 index 000000000..7414812ae --- /dev/null +++ b/docs/development/SFX_COMMANDS.md @@ -0,0 +1,203 @@ +# Studio SFX command contract + +`generation.sfx` is the version 2 command for the existing MMAudio sound +effects worker. It uses the shared command admission, receipt, task and +recovery path. Studio, Wizard and MCP share this runtime path. The visible SFX panel +acknowledges the exact command before admission; real model validation remains +separate acceptance work. + +## Envelope + +```json +{ + "version": 2, + "operation": "generation.sfx", + "intent_id": "sfx-intent-1", + "input": { + "workspace": "sfx-output", + "workspace_collection_id": "optional-collection-id", + "params": { + "model_type": "mmaudio_v2", + "prompt": "rain on a tin roof", + "MMAudio_neg_prompt": "speech, singing", + "duration_seconds": 5, + "seed": 20260909, + "guidance_scale": 4.5, + "sfx_text_weight": 1, + "video_guide": "/api/v1/file/guide.mp4?workspace=source" + } + } +} +``` + +The closed input accepts the registered virtual model IDs `mmaudio_v2` and +`mmaudio_nsfw`. `MMAudio_prompt` is accepted as the native spelling of +`prompt`; when both are supplied they must match exactly, including whitespace +and newlines. The optional `_mmaudio_variant` is checked against the selected +model and then derived again by the server. `MMAudio_setting` and `sfx_mode` +are inert legacy markers; the server recomputes them and does not treat them as +permission or download controls. + +The original envelope is detached before any lookup. The effective snapshot +contains deterministic native sentinels (`generation_mode=audio`, +`_audio_sub_mode=sfx`, `image_mode=0`, `video_length=0`) and the derived +MMAudio variant. The fingerprint covers the effective operation, workspace, +collection (when present) and parameters, while excluding `intent_id`. +Caller provenance, queue controls, provider payloads, remote URLs, host paths, +video carrier model IDs and download flags are rejected by the closed schema. + +## Duration and source behavior + +Text-only requests require a finite duration greater than zero and at most 20 +seconds. A video-guided request keeps its positive `duration_seconds` as the +requested control/provenance value, but preparation probes the canonical guide +and sets `duration_seconds_effective` and the worker `duration_seconds` to the +inspected guide duration. The two values must not be conflated. + +`video_guide` must be an exact asset ID or a canonical local URL under +`/api/v1/uploads/...` or `/api/v1/file/...?...workspace=...`. The declared +source workspace is resolved before file lookup. The guide must be a readable +video with positive finite duration and positive dimensions. Its durable +resource record includes URL, source workspace, SHA-256, byte size, duration, +dimensions and basic stream metadata; it never includes a host path. A missing +selected guide is an admission error and cannot silently become text-only. + +Text-only output is audio. Video-guided output is the original video with the +new MMAudio track, as implemented by the existing worker. The command does not +select or execute a video carrier model. + +## Pure preparation interface + +```python +prepare_studio_sfx( + params, + *, + model_definition, + model_downloaded, + resources, + execution_policy, +) +``` + +`model_definition(model_type)` and `model_downloaded(model_type)` are injected +catalog/install inspections. `model_downloaded` is the single required, +read-only dependency gate and must verify the complete installed-file set for +the selected model. A missing dependency produces `409`; no callback downloads +anything. The required files are exposed by +`required_mmaudio_files()` and match the files opened by `postprocessing.mmaudio`: +the selected v2 or NSFW weight, `mmaudio/v1-44.pth`, +`mmaudio/synchformer_state_dict.pth`, the DFN5B CLIP config/weights, and the +BigVGAN config/weights. + +`resources` is a `StudioSfxResources` instance. It resolves the canonical +guide, probes it and returns a detached worker map plus portable identities. +`execution_policy(workspace)` runs before model/resource inspection. All +callbacks are read-only; scheduling, receipt creation, downloads and native +inference belong to the shared runtime boundary. + +## Replay and recovery limits + +The durable receipt must retain the original/effective command, fingerprint, +variant and resource identities. Before native work, the execution guard must +match operation, intent, workspace, task ID, backend job ID and the prepared +parameters, then re-check the guide identity and installed dependency set. If a +guide disappears or changes, the request fails closed and is not converted to a +different source or mode. Reusing an intent is for replay/recovery of that same +admission; a changed request needs a new intent. + +This contract and its provider-free tests do not certify a GPU generation, +decoded WAV/MP4, or full Wizard/MCP presentation. Those require separate +runtime/UI acceptance evidence with an already-installed model. + + +## Runtime and visible Studio integration + +The runtime registers SFX against the existing `_run_generation` worker. It does +not generate a carrier video. Admission checks all seven installed dependencies +and the worker rechecks them without downloading. A guide is hashed again before +model work; this detects changes before that check but is not an immutable copy +or protection against a later filesystem replacement. + +The SFX panel displays the literal description, selected model, destination and +canonical guide. Without a guide it displays the requested duration; with a guide +it explains that output preserves the source duration and replaces its audio. +The selected source remains visible and removable after restoring settings. Its +source URL is retained even when the output belongs to another workspace. + +Recovery reuses the original command and receipt. The typed worker publishes +full parameter sidecars for WAV and video-guided MP4 output. Tests exercise the +real worker function with a provider stand-in, plus SQLite admissions, concurrent +retries, changed-resource failures and DOM acknowledgement. These checks do not +certify actual MMAudio inference or media decoding. + +## Wizard preparation + +`prepare_audio` with `audio_sub_mode=sfx` uses the same registered parser as +other audio actions. It retains prompt and negative prompt literally and +validates seed, guidance, 25 steps, one output and text weight. Preparation +checks the effective command before changing the form; it never compiles a +language-contract suffix into the sound description. + +Omitting `video_guide` retains the current selected guide. Explicit `null` +removes it. A nonempty canonical reference selects that source; empty action +strings and host/remote paths are rejected. The request duration is retained +until server preparation probes the guide. No-guidance requests are limited +to 20 seconds; the legacy SFX pack helper retains its separate clip behavior. + +### Restoring and submitting the SFX form + +The visible SFX description owns `MMAudio_prompt`. An empty description cannot +submit a leftover Speech/Music `prompt`. The direct command contract still accepts +a prompt-only envelope and rejects conflicting aliases. + +Loading an identified SFX output restores its literal description, negative prompt, +text weight and guidance, including explicit empty/zero values. Legacy SFX sidecars +with only `prompt` restore that text into the visible SFX field. Missing optional +SFX fields use native defaults rather than the previous clip's values. Recorded +audio duration takes precedence over video-frame conversion in Load Settings. + +### Switching into the SFX form + +Loading options for a virtual MMAudio model clears the previous model's options +locally and invalidates pending option requests. Boot, mode/model selection, +Wizard preparation and loading a sidecar use the same path. SFX must not inherit +video minimum durations or H3 Advanced controls, even when a late request succeeds +or fails. No MMAudio options or LoRA endpoint is fetched. + +The Wizard schema and instructions distinguish omitting `video_guide` (keep the +selected guide), an explicit `null` (clear), and a canonical reference (replace). +Replacing guide audio does not mean clearing the guide. This instructs the LLM; +it is not a deterministic guarantee of natural-language interpretation. Parser +and form tests verify each actual action's semantics separately from real runs. + +### Wizard packs and partial admission + +The Wizard pack uses one `generation.sfx` command per clip, with a distinct child +intent derived from the full parent command ID and the clip's ordered index. +Reusing that parent and input replays the same child receipts; a deliberate new +parent creates another pack, including separate clips with identical prompts. +An oversized parent that cannot fit the 160-character child intent limit fails +before form mutation. Ordinary Wizard-generated IDs fit this limit. + +Receipts, all admitted task IDs and pending child IDs survive a later failure as +a `partial` result. A receipt replay does not need to create a new UI tile. +Changing workspace stops subsequent clips. Presentation failure retains those +results and points to Activity. The registered runner retains the canonical +result and does not turn partial admission into completed media. Pack descriptions +and explicit empty negative prompts remain literal; invalid/oversized clip arrays +are rejected without dropping entries. The existing pack duration clamp to 1–20 s +remains separate from the single-SFX command contract. + +This does not add a server-side pack/workflow scheduler or persist automatic pack +advancement after the browser closes; those remain P9. Each admitted child itself +uses the shared durable queue and can be recovered independently through MCP. + +For an explicit SFX pack request, one complete JSON clip array on its own line +(optionally prefixed with `sfx_clips:` or `sfx_clips=`) is authoritative. Its +names, descriptions and order go through the registered pack parser even when +the LLM omits or rewrites its proposal. Explicit `negative_prompt="..."` JSON +strings and a single exact MMAudio model ID are retained. Multiple arrays or +conflicting declarations are rejected without guessing. An explicit pack with +no valid data must not fall through to Video merely because it mentions a guide. +This bounded source recovery does not interpret arbitrary prose as structured +clip data. diff --git a/docs/development/SHARED_NATIVE_COMMANDS.md b/docs/development/SHARED_NATIVE_COMMANDS.md new file mode 100644 index 000000000..622cb51c3 --- /dev/null +++ b/docs/development/SHARED_NATIVE_COMMANDS.md @@ -0,0 +1,140 @@ +# Use shared commands from Wizard or an MCP client + +Wizard operates inside the web application. MCP accepts explicit operations +from an external client while the application server is running. Closing a +browser does not stop an already admitted native generation or upscale job. +Server-driven editorial workflows are a separate migration and are not covered +by this guarantee. + +## Wizard and manual controls + +1. Choose the destination workspace and select installed models or existing + source media. For Tools, select the source, its kind and the upscale method. +2. Ask Wizard for the operation with those exact identifiers, or use the + ordinary Generate/Run controls. +3. The application presents the request in the relevant Studio/Tools panel. + A workspace or form change during preparation prevents that submission. +4. The queued message contains a task/job identity. Follow Activity to its + terminal status and open the resulting asset. Queued does not mean finished. +5. If a submission response is lost, recover the saved request in the panel. + Recovery reuses its intention; it must not silently create a new generation. + +Gallery Load Settings and Re-generate capture the clicked filename and source +workspace. They fetch that file's metadata directly instead of waiting 50 ms and +reading whichever item scrolling has selected. A newer restore request supersedes +an older one; a workspace change cancels a pending restore before reroll can +submit. Missing metadata cannot fall back to another clip's cached settings. + +The Generate button stays disabled while preparation/submission is pending and +shows “Preparing…”, never an optimistic “Queued” based on the click alone. Rapid +repeat clicks in that interval share the pending UI action. Admission and its +identity are reported by the command panel/Activity. Failed local placeholders +do not increase the active-job count on the button. + +The shared native routes currently cover image, speech, local music, SFX and upscale. Other +Studio modes continue through their existing paths until migrated. Speech +model duration controls have model-specific meanings: for example, a 20-second +Kugel setting does not force a short sentence to occupy exactly 20 seconds. +Music keeps the lyrics and Music Caption as distinct literal fields and does +not inherit speech voices. Its native model controls determine the accepted +duration; the Story song workflow retains its own contract. + +Audio reference selectors, source URLs, SFX video guides and displayed filenames belong to their +Speech, Music or SFX tab. Switching tabs starts an unused tab without inherited +audio references and restores that tab's own references when returning. Loading +an output's settings first retains the previous tab's references, then restores +the selected output's explicit references. Speech voice slots remain available +when returning to Speech; hidden voice counts cannot overwrite Music selectors +or cause new Music references to be discarded. These reference drafts live in +the current browser session; they do not add cross-reload draft persistence. +This isolation does not yet stash all prompts, captions or duration settings. +Direct command envelopes still reject incompatible active Speech metadata. + +## MCP connection and discovery + +Set `HOCUS_MCP_TOKEN` in the server's environment before starting the +application. Configure the external client's HTTP endpoint as +`http://SERVER:PORT/api/v1/wangp/mcp` and its Authorization header as +`Bearer YOUR_TOKEN`. Keep the actual token out of saved requests and reports. +The endpoint is disabled when no token is configured. + +The tested protocol is `2025-03-26`, using the official JavaScript MCP SDK's +Streamable HTTP client. Discovery uses `tools/list`; the returned schemas are +the authority for supported versions and parameters. The server does not invoke +Wizard's LLM to interpret an explicit MCP operation. + +Relevant tools include: + +| Tool | Purpose | +| --- | --- | +| `models`, `processors` | Discover exact model IDs and processor availability | +| `assets` | Find source IDs and workspace-qualified media URLs | +| `generation.image` | Submit the shared image specification | +| `generation.speech` | Submit the shared speech specification | +| `generation.music` | Submit literal lyrics and a music caption to an installed local model | +| `generation.sfx` | Generate MMAudio effects from text or replace a canonical video's audio | +| `tools.upscale` | Submit a typed upscale request for an existing image/video | +| `generation.receipt` | Recover a shared admission and its canonical task | +| `status` | Follow the returned native job ID | + +The previous `generate`, `upscale` and other legacy tools remain available. +Their schemas differ from the typed operations above; do not mix envelopes. + +For example, call `tools.upscale` with arguments shaped as follows, replacing +the example source with an existing resource obtained through `assets`: + +```json +{ + "version": 2, + "intent_id": "one-client-intention", + "input": { + "workspace": "my-outputs", + "params": { + "source": "/api/v1/file/source.png?workspace=my-inputs", + "source_kind": "image", + "method": "lanczos2", + "seed": 42, + "wangp_processor_settings": {} + } + } +} +``` + +MCP carries the operation in the tool name. The equivalent local HTTP request +adds `"operation": "tools.upscale"` and goes to +`POST /api/v1/generation/commands`. Its catalog is available at the same path +with GET. The collection ID, when supplied, is distinct from the physical +output workspace; see [Tools commands](TOOLS_COMMANDS.md). + +## Follow-up, retry and errors + +Save the command and its `intent_id` before sending. An admission response has +`receipt.result.job_id` and `receipt.result.task_id`. Poll `status` with the +job ID. On completion, its output filenames belong to the explicit destination +workspace; source media remain in their source workspace. + +To recover an uncertain response, call `generation.receipt` with: + +```json +{ + "version": 1, + "input": { + "workspace": "my-outputs", + "intent_id": "one-client-intention" + } +} +``` + +Repeat the exact command with the same intention after a transport failure. +An existing admission returns the same receipt and task. Changing the content +under that intention produces a conflict. A deliberately new generation uses +a new intention, even when the prompt is identical. After a server restart, +inspect the saved receipt and queue recovery before explicitly resuming an +interrupted job. + +Validation errors do not establish admission. A storage/dispatch error can +require recovery, so do not replace its intention automatically. Native task +status remains the completion authority. Read [image](IMAGE_COMMANDS.md), +[speech](SPEECH_COMMANDS.md), [music](MUSIC_COMMANDS.md), [SFX](SFX_COMMANDS.md) and [upscale](TOOLS_COMMANDS.md) contracts for +supported inputs and current limits. Hashes record inspected sources; they do +not make external source files immutable throughout queue lifetime. diff --git a/docs/development/SPEECH_COMMANDS.md b/docs/development/SPEECH_COMMANDS.md new file mode 100644 index 000000000..d0a8f62ea --- /dev/null +++ b/docs/development/SPEECH_COMMANDS.md @@ -0,0 +1,73 @@ +# Studio speech commands + +The shared command boundary for Studio speech is `version=2`, +`operation=generation.speech`: + +```json +{ + "version": 2, + "operation": "generation.speech", + "intent_id": "stable-client-intention", + "input": { + "workspace": "output-workspace", + "params": { + "prompt": "literal speech text", + "model_type": "kugelaudio_0_open", + "resolution": "1280x720", + "num_inference_steps": 0, + "guidance_scale": 3, + "seed": -1, + "generation_mode": "audio", + "image_mode": 0, + "video_length": 0, + "_audio_sub_mode": "speech", + "duration_seconds": 20 + } + } +} +``` + +`freeze_studio_speech_spec` validates and deep-copies the complete native +parameter map. The `original` snapshot keeps submitted spelling, omissions, +Unicode and multiline prompt text. The `effective` snapshot adds only the +speech selectors and other deterministic inactive/default values owned by the +adapter. It does not guess a model mode, duration, provider setting or voice +reference. The fingerprint is SHA-256 over version, operation, workspace and +effective parameters; `intent_id` and caller metadata are excluded so a retry +with another transport identity can be compared by content. + +The accepted speech model registration is deliberately explicit: + +`kugelaudio_0_open`, `qwen3_tts_customvoice`, `qwen3_tts_voicedesign`, +`qwen3_tts_base`, `chatterbox`, `index_tts2`, `scenema_audio`, and +`dramabox_audio`. The preflight requires the selected definition to declare +`audio_only=true`, no image output, and locally downloaded model files. Music +and SFX handlers also advertise audio-only and share the `tts` family, so +their metadata is not sufficient to enter this operation. + +Speech references use exact asset IDs or canonical local API URLs. The +resource adapter checks source-workspace containment, file identity and +positive audio format/duration metadata with `ffprobe` through the existing +`StudioSpeechResources` implementation; this preparation step does not claim +full audio decoding or copy the source. Host paths, traversal, remote URLs and +source-workspace overrides are rejected. + +The private `_tts_*` fields are typed aliases in the JSON contract. Speaker +names remain strings (or an inactive null), `_tts_voice_count` is an integer +from 0 through 6, and `_tts_original_prompt` remains separate from any +native speaker-tag rewrite. `audio_prompt_type` is checked against the +selected handler's declared choices; voice-count-driven handlers must match +their declared count mapping. `duration_seconds=0` is retained when the +selected model declares it as an auto-duration sentinel. + +`custom_settings` is a closed map of the currently registered speech setting +IDs (`auto_split_every_s`, `exaggeration`, `pace`, `vc_steps`, `vc_cfg_rate`, +and `duration_multiplier`). The selected model's own `custom_settings` +metadata supplies the allowed subset and numeric ranges. Native handler +validation remains the final authority for provider-specific prompt syntax, +speaker tags and generative settings. + +This slice intentionally does not announce music, SFX, video, avatar, 3D, +remote providers, arbitrary JSON options or a second scheduler. LoRA use is +accepted only when a speech definition explicitly declares +`enabled_audio_lora`; otherwise selected LoRAs fail before resource lookup. diff --git a/docs/development/TOOLS_COMMANDS.md b/docs/development/TOOLS_COMMANDS.md new file mode 100644 index 000000000..580c33fe9 --- /dev/null +++ b/docs/development/TOOLS_COMMANDS.md @@ -0,0 +1,71 @@ +# Tools upscale commands + +The first shared Tools operation is `tools.upscale`, version 2. It accepts the +same envelope used by the other shared commands: + +```json +{ + "version": 2, + "operation": "tools.upscale", + "intent_id": "intent-123", + "input": { + "workspace": "my-workspace", + "workspace_collection_id": "collection-123", + "params": { + "source": "/api/v1/file/poster.png?workspace=source", + "source_workspace": "source", + "source_kind": "image", + "method": "lanczos2", + "seed": -1, + "wangp_processor_settings": {} + } + } +} +``` + +`workspace` is the output workspace. An optional `workspace_collection_id` +identifies the logical collection associated with the command; it is retained +in the durable command snapshots, provenance and fingerprint and is not a native processor setting. `source` is either an exact asset ID +(`asset_...`) or one canonical local API reference: an upload URL, a +workspace-qualified file URL, or an exact asset URL. `source_workspace` is +optional; when supplied for a source URL it must match that URL's workspace. +Use `__uploads__` for an upload URL. For an asset ID with several catalog locations, omit the scope +only when there is exactly one location, otherwise select one exact +`source_workspace`. Absolute host paths, remote URLs, traversal, fragments, +missing scopes and ambiguous source locations are rejected. +The source kind must be `image` or `video`, and `method` is required rather +than selected by a default. + +The current method names come from the native Tools worker and are published +by `tools_upscale_schema()`. The preparation layer calls the existing WangGP +processor selection and settings validators. `WangpProcessorSettings` is the +only typed settings model; processor prompts retain their literal spelling. +The preparation adapter resolves processor reference images against their +explicit source workspace and records size and SHA-256 identity after Pillow +verification. The current native scalar-settings validator still rejects H3 +reference-image arrays; that option is not yet end-to-end supported. Image sources receive the same +verification. Video sources receive the existing `video_editor.probe_media` +probe and must report positive dimensions and duration. + +The freeze result keeps `original` detached and value-preserving. `effective` +adds only `seed=-1` and an empty processor-settings object when omitted. Its +fingerprint covers operation, output workspace and effective parameters while +excluding `intent_id` and caller metadata. The optional collection identity is +part of that fingerprint. The public receipt reports the physical output workspace; +collection identity is not a separate public receipt field. Resource identities are attached by the shared +submission service after preparation; the adapter does not create a queue or +write a journal. + +The adapter transfers the prepared request to the existing +`/api/v1/tools/upscale` native endpoint through an in-process callback. The +endpoint remains the authority for its established confined source resolver +and the existing `tools_upscale.py` worker remains the execution authority. +Real mode selects `_run_tool_upscale`; simulation mode selects the existing +`_run_generation` harness through the shared operation adapter. No model or +processor is downloaded by this command contract. + +This slice covers upscale only. Revoice, recast, background removal, music, +audio and model3d require their own explicit contracts and are not advertised +by this catalog entry. Resource preparation proves identity and media +inspectability; final output decoding and processor quality remain execution +QA responsibilities. diff --git a/scripts/export_music_command_catalog.py b/scripts/export_music_command_catalog.py new file mode 100644 index 000000000..6fb6080e2 --- /dev/null +++ b/scripts/export_music_command_catalog.py @@ -0,0 +1,39 @@ +"""Export the executable Studio music contract for its browser projection.""" + +from pathlib import Path +import argparse +import json +import sys + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "app")) +from routers.studio_music_commands import music_command_catalog +from services.studio_music_spec import studio_music_schema + + +def catalog(): + return { + "version": 2, + "operations": [music_command_catalog()], + "studio": studio_music_schema(), + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + expected = json.dumps(catalog(), indent=2, ensure_ascii=False) + "\n" + path = ROOT / "ui/src/api/musicCommandCatalog.json" + if args.check: + if not path.is_file() or path.read_text(encoding="utf-8") != expected: + raise SystemExit( + "Music command projection is stale; run scripts/export_music_command_catalog.py and review its diff" + ) + else: + path.write_text(expected, encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/scripts/export_sfx_command_catalog.py b/scripts/export_sfx_command_catalog.py new file mode 100644 index 000000000..8b8cc34f9 --- /dev/null +++ b/scripts/export_sfx_command_catalog.py @@ -0,0 +1,39 @@ +"""Export the executable Studio sfx contract for its browser projection.""" + +from pathlib import Path +import argparse +import json +import sys + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "app")) +from routers.studio_sfx_commands import sfx_command_catalog +from services.studio_sfx_spec import studio_sfx_schema + + +def catalog(): + return { + "version": 2, + "operations": [sfx_command_catalog()], + "studio": studio_sfx_schema(), + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + expected = json.dumps(catalog(), indent=2, ensure_ascii=False) + "\n" + path = ROOT / "ui/src/api/sfxCommandCatalog.json" + if args.check: + if not path.is_file() or path.read_text(encoding="utf-8") != expected: + raise SystemExit( + "SFX command projection is stale; run scripts/export_sfx_command_catalog.py and review its diff" + ) + else: + path.write_text(expected, encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/scripts/export_speech_command_catalog.py b/scripts/export_speech_command_catalog.py new file mode 100644 index 000000000..0c401f1b8 --- /dev/null +++ b/scripts/export_speech_command_catalog.py @@ -0,0 +1,28 @@ +"""Export the executable Studio speech contract for its browser projection.""" +from pathlib import Path +import argparse +import json +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "app")) +from routers.studio_speech_commands import speech_command_catalog +from services.studio_speech_spec import studio_speech_schema + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + value = {"version": 2, "operations": [speech_command_catalog()], "studio": studio_speech_schema()} + expected = json.dumps(value, indent=2, ensure_ascii=False) + "\n" + target = ROOT / "ui/src/api/speechCommandCatalog.json" + if args.check: + if not target.is_file() or target.read_text(encoding="utf-8") != expected: + raise SystemExit("Speech command projection is stale; regenerate and review its diff") + else: + target.write_text(expected, encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/scripts/export_tools_command_catalog.py b/scripts/export_tools_command_catalog.py new file mode 100644 index 000000000..85fae9924 --- /dev/null +++ b/scripts/export_tools_command_catalog.py @@ -0,0 +1,37 @@ +"""Export the executable Tools upscale contract consumed by the browser client.""" +from pathlib import Path +import argparse +import json +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "app")) +from routers.tools_upscale_commands import tools_upscale_command_catalog +from services.tools_upscale_spec import tools_upscale_schema + + +def catalog(): + return { + "version": 2, + "operations": [tools_upscale_command_catalog()], + "studio": tools_upscale_schema(), + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + path = ROOT / "ui/src/api/toolsCommandCatalog.json" + expected = json.dumps(catalog(), indent=2, ensure_ascii=False) + "\n" + if args.check: + if not path.is_file() or path.read_text(encoding="utf-8") != expected: + raise SystemExit( + "Tools command projection is stale; run scripts/export_tools_command_catalog.py and review its diff" + ) + else: + path.write_text(expected, encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/tests/fixtures/architecture_wire_inventory.json b/tests/fixtures/architecture_wire_inventory.json index 676485cf7..7016545f0 100644 --- a/tests/fixtures/architecture_wire_inventory.json +++ b/tests/fixtures/architecture_wire_inventory.json @@ -355,6 +355,12 @@ "classification": "fragile_source", "reason": "Python inspects TypeScript source; splitting useStore requires converting or relocating this contract." }, + { + "file": "tests/test_studio_sfx_native_worker.py", + "target": "app/_launch_runtime.py", + "classification": "symbol_importable", + "reason": "Extracts selected launch symbols with AST/exec; migrate to direct imports when that domain moves." + }, { "file": "tests/test_system_memory_profiles.py", "target": "app/_launch_runtime.py", @@ -373,6 +379,12 @@ "classification": "fragile_source", "reason": "Reads launch text directly and may need conversion when the referenced domain is extracted." }, + { + "file": "tests/test_tools_command_runtime.py", + "target": "app/_launch_runtime.py", + "classification": "symbol_importable", + "reason": "Extracts selected launch symbols with AST/exec; migrate to direct imports when that domain moves." + }, { "file": "tests/test_tools_upscale_contract.py", "target": "app/_launch_runtime.py", @@ -577,18 +589,48 @@ "classification": "behavior", "reason": "Imports the public Zustand facade; it should survive slice extraction unchanged." }, + { + "file": "ui/tests/studioAudioReferenceState.test.ts", + "target": "ui/src/stores/useStore.ts", + "classification": "behavior", + "reason": "Imports the public Zustand facade; it should survive slice extraction unchanged." + }, { "file": "ui/tests/studioEditPicker.test.tsx", "target": "ui/src/stores/useStore.ts", "classification": "behavior", "reason": "Imports the public Zustand facade; it should survive slice extraction unchanged." }, + { + "file": "ui/tests/studioMusicAction.test.mjs", + "target": "ui/src/stores/useStore.ts", + "classification": "behavior", + "reason": "Imports the public Zustand facade; it should survive slice extraction unchanged." + }, + { + "file": "ui/tests/studioMusicCommandPresentation.test.tsx", + "target": "ui/src/stores/useStore.ts", + "classification": "behavior", + "reason": "Imports the public Zustand facade; it should survive slice extraction unchanged." + }, { "file": "ui/tests/studioSectionsPicker.test.tsx", "target": "ui/src/stores/useStore.ts", "classification": "behavior", "reason": "Imports the public Zustand facade; it should survive slice extraction unchanged." }, + { + "file": "ui/tests/studioSfxAction.test.mjs", + "target": "ui/src/stores/useStore.ts", + "classification": "behavior", + "reason": "Imports the public Zustand facade; it should survive slice extraction unchanged." + }, + { + "file": "ui/tests/studioSfxCommandPresentation.test.tsx", + "target": "ui/src/stores/useStore.ts", + "classification": "behavior", + "reason": "Imports the public Zustand facade; it should survive slice extraction unchanged." + }, { "file": "ui/tests/tabFilterSearch.test.tsx", "target": "ui/src/stores/useStore.ts", @@ -607,6 +649,12 @@ "classification": "behavior", "reason": "Imports the public Zustand facade; it should survive slice extraction unchanged." }, + { + "file": "ui/tests/toolsUpscaleAdapter.test.ts", + "target": "ui/src/stores/useStore.ts", + "classification": "behavior", + "reason": "Imports the public Zustand facade; it should survive slice extraction unchanged." + }, { "file": "ui/tests/videoEditMultiClipPicker.test.tsx", "target": "ui/src/stores/useStore.ts", diff --git a/tests/fixtures/labs_wizard_action_matrix.json b/tests/fixtures/labs_wizard_action_matrix.json index afa4040a4..35d9cced8 100644 --- a/tests/fixtures/labs_wizard_action_matrix.json +++ b/tests/fixtures/labs_wizard_action_matrix.json @@ -2291,6 +2291,32 @@ "prompt_fixture": "", "blocking_defect": "" }, + { + "id": "other.upscale", + "lab": "other", + "surface": "upscale", + "user_operation": "Upscale one exact image or video source", + "control": "ToolsPanel / VideoInfoBar quick upscale", + "ui_handler": "ToolsPanel / VideoInfoBar.quickUpscaleClip", + "domain_function": "upscale", + "domain_module": "ui/src/features/agent/toolCapabilities.ts", + "adapter": "tools.upscale", + "api": "POST /api/v1/generation/commands", + "wizard_capability": "upscale", + "wizard_schema": "capabilityRegistry.upscale.inputSchema", + "in_wizard_context": false, + "wizard_status": "registrada_fuera_de_contexto", + "wizard_available": true, + "classification": "operativa", + "phase": "out_of_scope", + "preconditions": "Exact image/video source, installed upscale method and active workspace", + "persistence": "Durable command receipt and task", + "presentation": "tools.upscale", + "test": "ui/tests/toolsUpscaleAdapter.test.ts", + "notes": "The video shortcut enters Tools before the durable ACK presentation; it shares the same command gateway as the Tools button. Further coverage: ui/tests/toolsGenerationCommands.test.ts and ui/tests/toolsPanel.test.tsx. Outside the Story/Series Lab context.", + "prompt_fixture": "", + "blocking_defect": "" + }, { "id": "other.open_3d_scene", "lab": "other", diff --git a/tests/fixtures/studio_speech_native_request.json b/tests/fixtures/studio_speech_native_request.json new file mode 100644 index 000000000..518e98ec8 --- /dev/null +++ b/tests/fixtures/studio_speech_native_request.json @@ -0,0 +1,29 @@ +{ + "prompt": "The system is watching.\nEvery warning matters.", + "model_type": "kugelaudio_0_open", + "resolution": "1280x720", + "video_length": 0, + "num_inference_steps": 0, + "guidance_scale": 3, + "seed": -1, + "image_mode": 0, + "negative_prompt": "", + "repeat_generation": 1, + "activated_loras": [], + "loras_multipliers": "", + "settings_version": 2.52, + "flow_shift": 5, + "temperature": 1, + "audio_prompt_type": "", + "guidance_phases": 1, + "minimax_h3_turbo_mode": false, + "generation_mode": "audio", + "workspace": "speech-test", + "_audio_sub_mode": "speech", + "multi_prompts_gen_type": 2, + "_tts_original_prompt": "The system is watching.\nEvery warning matters.", + "_tts_speaker_name1": "", + "_tts_speaker_name2": "", + "_tts_voice_count": 0, + "duration_seconds": 20 +} diff --git a/tests/test_h3_preplan_job_contract.py b/tests/test_h3_preplan_job_contract.py index 1e3a38341..547d8fdb6 100644 --- a/tests/test_h3_preplan_job_contract.py +++ b/tests/test_h3_preplan_job_contract.py @@ -323,6 +323,7 @@ def acknowledge_cancel(job: dict, **updates) -> bool: "acknowledge_cancel": acknowledge_cancel, "snapshot_job": lambda job: dict(job), "_run_generation": lambda job_id, **_kwargs: gpu_calls.append(job_id), + "_image_generation_commands": SimpleNamespace(native_worker=lambda _job: None), } _load( "_new_generation_job", diff --git a/tests/test_studio_music_commands.py b/tests/test_studio_music_commands.py new file mode 100644 index 000000000..a33c4928c --- /dev/null +++ b/tests/test_studio_music_commands.py @@ -0,0 +1,174 @@ +"""Local music uses the same admission and queue, without a Story reservation.""" +from copy import deepcopy +from types import SimpleNamespace + +from fastapi import FastAPI, HTTPException +from fastapi.testclient import TestClient +import pytest + +from routers.image_generation_commands import create_image_generation_commands_router, image_command_handlers +from routers.studio_music_commands import music_command_catalog +from services.native_generation_operation import NativeGenerationOperation +from services.studio_music_spec import freeze_studio_music_spec +from services.image_generation_runtime import create_image_generation_commands +from services.wangp_submission import JsonRequest, prepare_generation_inputs +from tests.test_image_generation_commands import FakeNative, _command as image_command, _run +from tests.test_studio_music_preparation import ACE_DEFINITION +from tests.test_studio_speech_resources import write_wave + + +def music_command(intent="music-intent"): + return {"version": 2, "operation": "generation.music", "intent_id": intent, + "input": {"workspace": "music-test", "params": { + "model_type": "ace_step_v1_5_xl_sft_lm_4b", + "prompt": " [Verse]\nThe city listens\nEvery light replies\n ", + "alt_prompt": " Soft drums, luminous synths.\nA quiet chorus. ", + "_music_description": " A supervisor watches the city. ", + "_music_instrumental": False, + "duration_seconds": 20, "seed": 42, + "num_inference_steps": 8, "guidance_scale": 7.0, + "resolution": "1280x720", "generation_mode": "audio", + "image_mode": 0, "video_length": 0, "_audio_sub_mode": "music", + }}} + + +def music_service(native): + service = native.service() + native_prepare = service.prepare + + async def prepare_request(request): + assert request.prepared_studio_audio is True + assert request.prepared_studio_speech is False + return await native_prepare(request) + + def freeze(command): + frozen = freeze_studio_music_spec(command) + effective = frozen["effective"]["input"] + return frozen, {**deepcopy(effective["params"]), "workspace": effective["workspace"]} + + service.prepare = prepare_request + service.operations["generation.music"] = NativeGenerationOperation( + freeze=freeze, prepare=lambda params: (params, []), catalog=music_command_catalog(), + ) + return service + + +def test_local_music_http_and_mcp_share_literal_request_and_admission(tmp_path): + native = FakeNative(tmp_path) + service = music_service(native) + app = FastAPI() + app.include_router(create_image_generation_commands_router(service)) + command = music_command() + with TestClient(app) as client: + response = client.post('/api/v1/generation/commands', json=command, + headers={"X-Hocus-UI-Surface": "wizard"}) + assert response.status_code == 200, response.text + reply = _run(image_command_handlers(service)["generation.music"]( + {key: value for key, value in command.items() if key != "operation"})) + assert reply == {"receipt": response.json()["receipt"], "replayed": True} + assert len(native.dispatch_calls) == 1 + job = native.dispatch_calls[0] + for key in ("prompt", "alt_prompt", "_music_description", "duration_seconds", "seed"): + assert job["params"][key] == command["input"]["params"][key] + assert job["provenance"]["actor"] == "wizard" + assert job["provenance"]["capability"] == "generation.music" + assert reply["receipt"]["result"]["workspace"] == "music-test" + entry = native.registry("music-test").command_admission(command["intent_id"]) + assert entry["original"] == command + assert entry["effective"]["runtime"]["params"]["prompt"] == command["input"]["params"]["prompt"] + + +def test_new_music_intention_is_distinct_but_reused_intention_rejects_changed_lyrics(tmp_path): + native = FakeNative(tmp_path) + service = music_service(native) + first = _run(service.submit(music_command())) + altered = music_command() + altered["input"]["params"]["prompt"] += "Another line" + with pytest.raises(HTTPException) as conflict: + _run(service.submit(altered)) + assert conflict.value.status_code == 409 + with pytest.raises(HTTPException) as domain_conflict: + _run(service.submit(image_command("music-intent", workspace="music-test"))) + assert domain_conflict.value.status_code == 409 + second = _run(service.submit(music_command("deliberate-second-song"))) + assert first["receipt"]["taskIds"] != second["receipt"]["taskIds"] + assert len(native.dispatch_calls) == 2 + + +def test_music_recovery_keeps_original_lyrics_and_job_without_dispatch(tmp_path): + native = FakeNative(tmp_path) + first = _run(music_service(native).submit(music_command())) + restarted = FakeNative(tmp_path, interrupt_stale=True) + service = music_service(restarted) + service.restore_recovery(["music-test"]) + assert restarted.dispatch_calls == [] + record = restarted.persist_calls[0] + assert record["params"]["prompt"] == music_command()["input"]["params"]["prompt"] + assert record["id"] == first["receipt"]["result"]["job_id"] + assert service.filter_recovery([record]) == [record] + assert _run(service.submit(music_command()))["receipt"] == first["receipt"] + assert restarted.dispatch_calls == [] + + +def test_music_factory_preserves_audio_origin_through_native_preparation(tmp_path): + native = FakeNative(tmp_path) + for folder in ("music-test", "reference", "uploads"): + (tmp_path / folder).mkdir() + source = tmp_path / "reference" / "music.wav" + write_wave(source, 800) + write_wave(tmp_path / "music-test" / "music.wav", 1600) + observed = {} + + async def native_generate(request): + body = await request.json() + assert request.prepared_studio_audio is True + assert request.prepared_studio_speech is False + prepare_generation_inputs( + body, ACE_DEFINITION, body["workspace"], + uploads_dir=tmp_path / "uploads", workspace_dir=tmp_path / body["workspace"], + prepared_speech=request.prepared_studio_audio is True, + ) + observed.update(deepcopy(body)) + return await native.prepare(JsonAdmissionRequest(body, request.admit_generation_command)) + + runtime = { + "_task_registry": native.registry, "generate": native_generate, + "_new_generation_job": native.make_job, + "_generation_task_fields": native.service().task_fields, + "_jobs": {}, "_check_model_downloaded": lambda _model: True, + "_workspace_dir": lambda workspace: str(tmp_path / workspace), + "_list_workspaces": lambda: [{"name": "reference"}, {"name": "music-test"}], + "_lora_is_compatible_with_model": lambda *_args: False, + "wgp": SimpleNamespace(primary_settings={}, get_model_def=lambda _: deepcopy(ACE_DEFINITION), + get_lora_search_dirs=lambda _: []), + "execution_mode": SimpleNamespace(validate_generation=lambda _: None), + } + service = create_image_generation_commands(runtime) + # Stop at the existing dispatch seam; resource inspection and the runtime + # factory are real, but this test does not load a model or start a worker. + service.dispatch = native.dispatch + service.persist_recovery = native.persist_recovery + command = music_command() + command["input"]["params"].update(audio_prompt_type="A", + audio_guide="/api/v1/file/music.wav?workspace=reference") + reply = _run(service.submit(command)) + assert observed["audio_guide"] == str(source) + assert observed["prompt"] == command["input"]["params"]["prompt"] + assert len(native.dispatch_calls) == 1 + entry = native.registry("music-test").command_admission(command["intent_id"]) + resource = entry["effective"]["resources"][0] + assert resource["workspace"] == "reference" + assert resource["duration_seconds"] == 0.1 + assert reply["receipt"]["operation"] == "generation.music" + assert _run(service.submit(command))["replayed"] is True + assert len(native.dispatch_calls) == 1 + + +class JsonAdmissionRequest(JsonRequest): + def __init__(self, body, admit): + super().__init__(body) + self.admit_generation_command = admit + + +def test_music_preparation_marker_cannot_be_forged_by_json(): + assert getattr(JsonRequest({"prepared_studio_audio": True}), "prepared_studio_audio", False) is False diff --git a/tests/test_studio_music_preparation.py b/tests/test_studio_music_preparation.py new file mode 100644 index 000000000..3b2d77dd2 --- /dev/null +++ b/tests/test_studio_music_preparation.py @@ -0,0 +1,332 @@ +"""Provider-free tests for Studio music model/resource preparation.""" + +from copy import deepcopy +import pytest +from fastapi import HTTPException + +from services.studio_music_preparation import prepare_studio_music +from services.studio_music_spec import freeze_studio_music_spec + + +ACE_DEFINITION = { + "audio_only": True, + "image_outputs": False, + "guidance_max_phases": 1, + "no_negative_prompt": True, + "inference_steps": True, + "temperature": True, + "top_p_slider": True, + "top_k_slider": True, + "audio_scale_name": "Source Audio Strength", + "alt_guidance": "LM Guidance", + "enabled_audio_lora": True, + "duration_slider": {"min": 5, "max": 360, "default": 120}, + "audio_prompt_type_sources": {"selection": ["", "A", "B", "AB"], "default": ""}, + "custom_settings": [ + {"id": "bpm", "type": "int", "min": 30, "max": 300}, + {"id": "keyscale", "type": "text"}, + {"id": "timesignature", "type": "int", "min": 2, "max": 6}, + {"id": "language", "type": "text"}, + ], +} + +MUSIC3_DEFINITION = { + "audio_only": True, + "image_outputs": False, + "guidance_max_phases": 0, + "lock_guidance_scale": True, + "no_negative_prompt": True, + "inference_steps": True, + "temperature": False, + "duration_slider": {"min": 5, "max": 300, "default": 120}, +} + + +def command_params(model_type="ace_step_v1_5_xl_sft_lm_4b", **overrides): + params = { + "workspace": "music-test", + "model_type": model_type, + "prompt": "[Verse]\nKeep the literal line", + "alt_prompt": "Warm acoustic pop", + "generation_mode": "audio", + "_audio_sub_mode": "music", + "image_mode": 0, + "video_length": 0, + "duration_seconds": 20, + "num_inference_steps": 8 if model_type != "minimax_music3" else 30, + "guidance_scale": 1.0 if model_type != "minimax_music3" else 1.7, + "seed": 7, + } + params.update(overrides) + return params + + +class FakeResources: + def __init__(self, *, media_result=None, lora_result=None, media_error=None): + self.media_result = media_result + self.lora_result = [] if lora_result is None else lora_result + self.media_error = media_error + self.media_calls = [] + self.lora_calls = [] + + def prepare_media(self, params): + self.media_calls.append(deepcopy(params)) + if self.media_error is not None: + raise self.media_error + if self.media_result is not None: + return deepcopy(self.media_result) + return deepcopy(params), [] + + def prepare_loras(self, params, definition): + self.lora_calls.append((deepcopy(params), deepcopy(definition))) + return deepcopy(self.lora_result) + + +def invoke(params, *, definition=None, downloaded=True, resources=None, policy_error=None): + definition = deepcopy(definition or ACE_DEFINITION) + resources = resources or FakeResources() + calls = {"policy": [], "definition": [], "downloaded": []} + + def model_definition(model_type): + calls["definition"].append(model_type) + return deepcopy(definition) + + def model_downloaded(model_type): + calls["downloaded"].append(model_type) + return downloaded + + def policy(workspace): + calls["policy"].append(workspace) + if policy_error is not None: + raise policy_error + + result = prepare_studio_music( + params, + model_definition=model_definition, + model_downloaded=model_downloaded, + resources=resources, + execution_policy=policy, + ) + return result, calls, resources + + +def frozen_params(**overrides): + native = command_params(**overrides) + native.pop("workspace", None) + command = { + "version": 2, + "operation": "generation.music", + "intent_id": "music-intent", + "input": {"workspace": "music-test", "params": native}, + } + frozen = freeze_studio_music_spec(command) + return {**frozen["effective"]["input"]["params"], "workspace": "music-test"} + + +def raw_params(**overrides): + params = command_params(**overrides) + params["workspace"] = "music-test" + params.pop("_audio_sub_mode", None) + return params + + +def test_installed_ace_returns_model_defaults_and_detached_resources(): + params = frozen_params() + before = deepcopy(params) + resources = FakeResources( + media_result=({**params, "native_marker": {"kept": True}}, [{"role": "audio_guide"}]), + lora_result=[{"role": "lora", "name": "music-style"}], + ) + (native, identities), calls, _ = invoke(params, resources=resources) + + assert params == before + assert native["native_marker"] == {"kept": True} + assert identities == [{"role": "audio_guide"}, {"role": "lora", "name": "music-style"}] + assert calls["policy"] == ["music-test"] + assert calls["definition"] == ["ace_step_v1_5_xl_sft_lm_4b"] + assert calls["downloaded"] == ["ace_step_v1_5_xl_sft_lm_4b"] + native["native_marker"]["kept"] = False + assert resources.media_result[0]["native_marker"] == {"kept": True} + + +@pytest.mark.parametrize("model_type", ["music-3.0", "minimax_music3_gguf", "unknown"]) +def test_remote_community_and_unknown_models_fail_before_resource_inspection(model_type): + resources = FakeResources() + with pytest.raises(HTTPException) as error: + invoke(raw_params(model_type=model_type), resources=resources) + assert error.value.status_code == 422 + assert resources.media_calls == resources.lora_calls == [] + + +@pytest.mark.parametrize( + ("definition", "downloaded", "status"), + [({**ACE_DEFINITION, "audio_only": False}, True, 422), + ({**ACE_DEFINITION, "image_outputs": True}, True, 422), + (ACE_DEFINITION, False, 409)], +) +def test_model_must_be_audio_only_compatible_and_installed(definition, downloaded, status): + resources = FakeResources() + with pytest.raises(HTTPException) as error: + invoke(frozen_params(), definition=definition, downloaded=downloaded, resources=resources) + assert error.value.status_code == status + assert resources.media_calls == resources.lora_calls == [] + + +def test_execution_policy_runs_before_model_or_resource_lookup(): + resources = FakeResources() + with pytest.raises(HTTPException) as error: + invoke( + frozen_params(), resources=resources, + policy_error=HTTPException(409, {"code": "execution_policy", "message": "busy"}), + ) + assert error.value.status_code == 409 + assert resources.media_calls == resources.lora_calls == [] + + +@pytest.mark.parametrize( + ("model_type", "definition"), + [("ace_step_v1_5_xl_sft_lm_4b", ACE_DEFINITION), ("minimax_music3", MUSIC3_DEFINITION)], +) +def test_duration_uses_native_slider_bounds_not_story_minimum(model_type, definition): + params = frozen_params(model_type=model_type) + params.pop("duration_seconds") + (native, _), _, _ = invoke(params, definition=definition) + assert native["duration_seconds"] == 120 + + short = frozen_params(model_type=model_type, duration_seconds=5) + (native, _), _, _ = invoke(short, definition=definition) + assert native["duration_seconds"] == 5 + + with pytest.raises(HTTPException) as error: + invoke(frozen_params(model_type=model_type, duration_seconds=4), definition=definition) + assert "duration_seconds" in error.value.detail["message"] + + maximum = definition["duration_slider"]["max"] + with pytest.raises(HTTPException) as error: + invoke(frozen_params(model_type=model_type, duration_seconds=maximum + 1), definition=definition) + assert "duration_seconds" in error.value.detail["message"] + + +def test_missing_duration_slider_uses_native_music_minimum(): + definition = deepcopy(ACE_DEFINITION) + definition.pop("duration_slider") + + with pytest.raises(HTTPException) as error: + invoke(frozen_params(duration_seconds=4), definition=definition) + assert "duration_seconds" in error.value.detail["message"] + + (native, _), _, _ = invoke(frozen_params(duration_seconds=5), definition=definition) + assert native["duration_seconds"] == 5 + + +@pytest.mark.parametrize("field", ["num_inference_steps", "guidance_scale"]) +def test_nonfinite_or_invalid_sampling_fails_before_media(field): + bad = raw_params(**{field: float("nan")}) if field == "guidance_scale" else raw_params(**{field: 0}) + resources = FakeResources() + with pytest.raises(HTTPException) as error: + invoke(bad, resources=resources) + assert error.value.status_code == 422 + assert resources.media_calls == [] + + +def test_music3_uses_its_defaults_and_rejects_unsupported_sampling(): + params = frozen_params(model_type="minimax_music3") + params.pop("num_inference_steps") + params.pop("guidance_scale") + (native, _), _, _ = invoke(params, definition=MUSIC3_DEFINITION) + assert native["num_inference_steps"] == 30 + assert native["guidance_scale"] == 1.7 + with pytest.raises(HTTPException) as error: + invoke(frozen_params(model_type="minimax_music3", temperature=0.5), definition=MUSIC3_DEFINITION) + assert "temperature" in error.value.detail["message"] + + +def test_ace_audio_selector_requires_canonical_reference_slots(): + missing = frozen_params(audio_prompt_type="A") + with pytest.raises(HTTPException) as error: + invoke(missing) + assert "audio_guide" in error.value.detail["message"] + + orphan = frozen_params(audio_guide="/api/v1/uploads/track.wav") + with pytest.raises(HTTPException) as error: + invoke(orphan) + assert "audio_prompt_type" in error.value.detail["message"] + + valid = frozen_params( + audio_prompt_type="AB", + audio_guide="/api/v1/uploads/one.wav", + audio_guide2="/api/v1/file/two.wav?workspace=music-test", + ) + resources = FakeResources() + (native, _), _, _ = invoke(valid, resources=resources) + assert native["audio_guide"] == valid["audio_guide"] + assert native["audio_guide2"] == valid["audio_guide2"] + + +def test_music3_rejects_reference_audio_and_ace_rejects_extra_slots(): + with pytest.raises(HTTPException) as error: + invoke( + frozen_params(model_type="minimax_music3", audio_guide="/api/v1/uploads/song.wav"), + definition=MUSIC3_DEFINITION, + ) + assert "reference audio" in error.value.detail["message"] + + with pytest.raises(HTTPException) as error: + invoke(frozen_params(audio_prompt_type="AB", audio_guide3="/api/v1/uploads/third.wav")) + assert "audio_guide3" in error.value.detail["message"] + + +def test_music3_empty_reference_sentinels_are_normalized_for_native_handler(): + params = frozen_params(model_type="minimax_music3", audio_guide="", audio_guide2="") + (native, _), _, _ = invoke(params, definition=MUSIC3_DEFINITION) + assert native["audio_guide"] is None + assert native["audio_guide2"] is None + + +def test_language_guard_reuses_music_contract_without_rewriting_lyrics(): + params = frozen_params(lyrics_language="es", prompt="[Verse]\nThe night is singing through the server") + with pytest.raises(HTTPException) as error: + invoke(params) + assert "idioma" in error.value.detail["message"] + + valid = frozen_params(lyrics_language="es", prompt="[Verse]\nLa noche canta") + (native, _), _, _ = invoke(valid) + assert native["prompt"] == valid["prompt"] + + +def test_instrumental_flag_cannot_hide_vocal_lyrics(): + params = frozen_params(_music_instrumental=True, prompt="A vocal line") + with pytest.raises(HTTPException) as error: + invoke(params) + assert "instrumental" in error.value.detail["message"].lower() + + valid = frozen_params(_music_instrumental=True, prompt="[Instrumental]") + invoke(valid) + + +def test_custom_settings_use_model_metadata_and_unknown_keys_fail_closed(): + valid = frozen_params(custom_settings={"bpm": 120, "keyscale": "C major", "timesignature": 4, "language": "en"}) + (native, _), _, _ = invoke(valid) + assert native["custom_settings"]["bpm"] == 120 + + for values in ({"bpm": 29}, {"unknown": 1}): + with pytest.raises(HTTPException) as error: + invoke(raw_params(custom_settings=values)) + assert "custom_settings" in error.value.detail["message"] + + inactive = frozen_params(custom_settings={"bpm": "", "timesignature": ""}) + (native, _), _, _ = invoke(inactive) + assert native["custom_settings"] == {"bpm": "", "timesignature": ""} + + +def test_loras_require_declared_capability_and_resource_errors_are_wrapped(): + params = frozen_params(activated_loras=["music-style"]) + with pytest.raises(HTTPException) as error: + invoke(params, definition={**ACE_DEFINITION, "enabled_audio_lora": False}) + assert "lora" in error.value.detail["message"].lower() + + resources = FakeResources(media_error=ValueError("bad music reference")) + with pytest.raises(HTTPException) as error: + invoke(frozen_params(), resources=resources) + assert "bad music reference" in error.value.detail["message"] + assert resources.lora_calls == [] diff --git a/tests/test_studio_music_spec.py b/tests/test_studio_music_spec.py new file mode 100644 index 000000000..7c773c4d8 --- /dev/null +++ b/tests/test_studio_music_spec.py @@ -0,0 +1,257 @@ +"""Provider-free tests for the closed Studio music command.""" + +from copy import deepcopy +import hashlib +import json + +import pytest + +from services.studio_music_spec import ( + STUDIO_MUSIC_DEFAULTS, + SUPPORTED_INPUT_FIELDS, + StudioMusicSpecError, + freeze_studio_music_spec, + studio_music_schema, +) + + +def music_command(intent="music-intent"): + return { + "version": 2, + "operation": "generation.music", + "intent_id": intent, + "input": { + "workspace": "music-test", + "workspace_collection_id": "collection-a", + "params": { + "model_type": "ace_step_v1_5_xl_sft_lm_4b", + "prompt": " [Verse]\nA literal line\n ", + "alt_prompt": " warm acoustic pop\nwith brushed drums ", + "_music_description": " the description remains literal ", + "_music_instrumental": False, + "duration_seconds": 20, + "seed": 42, + "num_inference_steps": 8, + "guidance_scale": 1.0, + "resolution": "1280x720", + "generation_mode": "audio", + "image_mode": 0, + "video_length": 0, + "_audio_sub_mode": "music", + }, + }, + } + + +def test_freeze_preserves_literal_music_text_and_detaches_input(): + command = music_command() + command["input"]["params"]["lyrics_language"] = "es-MX" + before = deepcopy(command) + frozen = freeze_studio_music_spec(command) + + assert command == before + assert frozen["original"] == before + assert frozen["original"] is not command + assert frozen["original"]["input"]["params"]["prompt"] == before["input"]["params"]["prompt"] + assert frozen["effective"]["input"]["params"]["prompt"] == before["input"]["params"]["prompt"] + assert frozen["effective"]["input"]["params"]["alt_prompt"] == before["input"]["params"]["alt_prompt"] + assert frozen["effective"]["input"]["params"]["_music_description"] == before["input"]["params"]["_music_description"] + assert frozen["effective"]["input"]["params"]["lyrics_language"] == "es-MX" + assert frozen["effective"]["input"]["workspace_collection_id"] == "collection-a" + + command["input"]["params"]["prompt"] = "changed" + assert frozen["original"]["input"]["params"]["prompt"] == before["input"]["params"]["prompt"] + + +def test_effective_defaults_are_audio_only_and_omitted_bookkeeping_is_literal(): + command = music_command() + params = command["input"]["params"] + for key in ( + "generation_mode", + "_audio_sub_mode", + "video_length", + "image_mode", + "multi_prompts_gen_type", + "negative_prompt", + "repeat_generation", + "batch_size", + "activated_loras", + "loras_multipliers", + "prompt_enhancer", + "_music_description", + "_music_instrumental", + ): + params.pop(key, None) + frozen = freeze_studio_music_spec(command) + effective = frozen["effective"]["input"]["params"] + + for key, value in STUDIO_MUSIC_DEFAULTS.items(): + assert effective[key] == value + assert effective["_tts_original_prompt"] == params["prompt"] + assert "_tts_original_prompt" not in params + + +def test_instrumental_marker_remains_literal_and_is_not_rewritten(): + command = music_command() + command["input"]["params"].update({"prompt": " [Instrumental]\n ", "_music_instrumental": True}) + + frozen = freeze_studio_music_spec(command) + + assert frozen["original"]["input"]["params"]["prompt"] == " [Instrumental]\n " + assert frozen["effective"]["input"]["params"]["prompt"] == " [Instrumental]\n " + assert frozen["effective"]["input"]["params"]["_music_instrumental"] is True + assert frozen["effective"]["input"]["params"]["alt_prompt"] == command["input"]["params"]["alt_prompt"] + + +def test_empty_or_omitted_alt_prompt_is_admitted(): + blank = music_command() + blank["input"]["params"]["alt_prompt"] = "" + frozen_blank = freeze_studio_music_spec(blank) + assert frozen_blank["effective"]["input"]["params"]["alt_prompt"] == "" + + omitted = music_command() + omitted["input"]["params"].pop("alt_prompt") + frozen_omitted = freeze_studio_music_spec(omitted) + assert "alt_prompt" not in frozen_omitted["original"]["input"]["params"] + assert frozen_omitted["effective"]["input"]["params"].get("alt_prompt", "") == "" + + +def test_fingerprint_excludes_intent_but_covers_workspace_collection_and_content(): + first = freeze_studio_music_spec(music_command("first")) + second = freeze_studio_music_spec(music_command("second")) + assert first["fingerprint"] == second["fingerprint"] + + changed_collection = music_command("third") + changed_collection["input"]["workspace_collection_id"] = "collection-b" + assert freeze_studio_music_spec(changed_collection)["fingerprint"] != first["fingerprint"] + + changed_prompt = music_command("fourth") + changed_prompt["input"]["params"]["prompt"] += " added" + assert freeze_studio_music_spec(changed_prompt)["fingerprint"] != first["fingerprint"] + + +def test_fingerprint_uses_canonical_json_without_transport_identity(): + frozen = freeze_studio_music_spec(music_command()) + content = { + "version": 2, + "operation": "generation.music", + "input": frozen["effective"]["input"], + } + expected = hashlib.sha256( + json.dumps(content, ensure_ascii=False, sort_keys=True, + separators=(",", ":"), allow_nan=False).encode() + ).hexdigest() + assert frozen["fingerprint"] == expected + assert len(frozen["fingerprint"]) == 64 + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("generation_mode", "image"), + ("_audio_sub_mode", "speech"), + ("image_mode", 1), + ("video_length", 1), + ("multi_prompts_gen_type", 1), + ("_tts_speaker_name1", "Alice"), + ("_tts_voice_count", 1), + ("negative_prompt", "avoid drums"), + ("prompt_enhancer", "rewrite this"), + ("audio_prompt_type", "N"), + ], +) +def test_music_rejects_active_other_mode_fields(field, value): + command = music_command() + command["input"]["params"][field] = value + with pytest.raises(StudioMusicSpecError): + freeze_studio_music_spec(command) + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("num_inference_steps", True), + ("num_inference_steps", 1.0), + ("seed", 1.5), + ("guidance_scale", True), + ("duration_seconds", "20"), + ("duration_seconds", float("nan")), + ("_music_instrumental", 1), + ("_tts_voice_count", True), + ("custom_settings", {"unknown": 1}), + ], +) +def test_music_native_scalars_and_nested_fields_are_strict(field, value): + command = music_command() + command["input"]["params"][field] = value + with pytest.raises(StudioMusicSpecError): + freeze_studio_music_spec(command) + + +@pytest.mark.parametrize( + "value", + [ + "/etc/music.wav", + "../music.wav", + "C:\\music.wav", + "https://example.test/music.wav", + "/api/v1/file/music.wav", + "/api/v1/file/../music.wav?workspace=music-test", + "/api/v1/uploads/music.wav?workspace=other", + ], +) +def test_music_references_must_be_canonical(value): + command = music_command() + command["input"]["params"].update({"audio_prompt_type": "A", "audio_guide": value}) + with pytest.raises(StudioMusicSpecError): + freeze_studio_music_spec(command) + + +@pytest.mark.parametrize("value", ["", None, "asset_abc123", "/api/v1/uploads/music.wav"]) +def test_music_reference_sentinels_and_asset_urls_are_retained(value): + command = music_command() + command["input"]["params"].update({"audio_prompt_type": "", "audio_guide": value}) + frozen = freeze_studio_music_spec(command) + assert frozen["effective"]["input"]["params"]["audio_guide"] == value + + +def test_music_models_are_explicit_and_remote_ids_do_not_enter_local_command(): + for model in ("music-3.0", "minimax_music3_gguf", "unknown"): + command = music_command() + command["input"]["params"]["model_type"] = model + with pytest.raises(StudioMusicSpecError): + freeze_studio_music_spec(command) + + +def test_schema_is_closed_and_exposes_only_local_models(): + schema = studio_music_schema() + assert schema["version"] == 2 + assert schema["operation"] == "generation.music" + params = schema["input"]["$defs"]["StudioMusicParams"] + assert params["additionalProperties"] is False + assert "_music_description" in params["properties"] + assert schema["music_model_types"] == ["ace_step_v1_5_xl_sft_lm_4b", "minimax_music3"] + assert "provenance" in schema["excluded"] + + +def test_supported_input_fields_identify_every_closed_native_property_once(): + schema = studio_music_schema() + properties = tuple(schema["input"]["$defs"]["StudioMusicParams"]["properties"]) + supported = tuple(SUPPORTED_INPUT_FIELDS) + + assert set(supported) == set(properties) + assert len(supported) == len(set(supported)) + assert set(schema["supported_input_fields"]) == set(properties) + assert "lyrics_language" in properties + + +@pytest.mark.parametrize("caption", ["", " ", None]) +def test_music3_still_requires_caption_before_admission(caption): + command = music_command() + command["input"]["params"]["model_type"] = "minimax_music3" + if caption is None: + command["input"]["params"].pop("alt_prompt") + else: + command["input"]["params"]["alt_prompt"] = caption + with pytest.raises(StudioMusicSpecError, match="alt_prompt"): + freeze_studio_music_spec(command) diff --git a/tests/test_studio_sfx_commands.py b/tests/test_studio_sfx_commands.py new file mode 100644 index 000000000..a49594e78 --- /dev/null +++ b/tests/test_studio_sfx_commands.py @@ -0,0 +1,139 @@ +"""SFX uses canonical admissions and the existing queue, with no model calls.""" +from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy +from types import SimpleNamespace + +from fastapi import FastAPI, HTTPException +from fastapi.testclient import TestClient +import pytest + +from routers.image_generation_commands import create_image_generation_commands_router, image_command_handlers +from services.studio_sfx_commands import create_sfx_operation, check_sfx_models +from services.studio_sfx_execution import prepared_sfx_execution +from services.studio_sfx_resources import StudioSfxResources, required_mmaudio_files +from tests.test_image_generation_commands import FakeNative, _run + + +class SfxNative(FakeNative): + def make_job(self, *args, **kwargs): + job = super().make_job(*args, **kwargs) + job["task_id"] = f"task-{job['id']}" + return job + + +def command(intent="sound-intent"): + return {"version": 2, "operation": "generation.sfx", "intent_id": intent, + "input": {"workspace": "sound-output", "params": { + "model_type": "mmaudio_v2", "prompt": " Rain against glass.\n ", + "duration_seconds": 3, "seed": 42, + }}} + + +def setup_service(tmp_path): + native = SfxNative(tmp_path / "registry") + weights = tmp_path / "weights" + for name in required_mmaudio_files("v2"): + path = weights / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"installed test file; no model is loaded") + runtime = {"wgp": SimpleNamespace(fl=SimpleNamespace( + locate_file=lambda name, **_kwargs: str(weights / name) if (weights / name).exists() else None, + )), "_run_generation": lambda _job_id: True} + resources = StudioSfxResources( + workspace_dir=lambda name: str(tmp_path / name), uploads_dir=lambda: str(tmp_path / "uploads"), + list_workspaces=lambda: [{"name": name} for name in ("source", "sound-output", "default")], + lora_search_dirs=lambda _model: [], lora_compatible=lambda *_args: False, + ) + service = native.service() + service.runtime_defaults = lambda: {"model_type": "wrong-video-model", "prompt": "global residue"} + service.operations["generation.sfx"] = create_sfx_operation( + runtime, resources=lambda: resources, execution_policy=lambda _workspace: None, + ) + return native, service, runtime, weights + + +def test_http_and_external_mcp_replay_share_one_literal_native_admission(tmp_path): + native, service, runtime, _ = setup_service(tmp_path) + app = FastAPI() + app.include_router(create_image_generation_commands_router(service)) + request = command() + with TestClient(app) as client: + first = client.post('/api/v1/generation/commands', json=request, + headers={"X-Hocus-UI-Surface": "wizard"}) + assert first.status_code == 200, first.text + replay = _run(image_command_handlers(service)['generation.sfx']( + {key: value for key, value in request.items() if key != 'operation'})) + assert replay == {"receipt": first.json()["receipt"], "replayed": True} + assert len(native.dispatch_calls) == 1 + job = native.dispatch_calls[0] + assert job['params']['model_type'] == 'mmaudio_v2' + assert job['params']['prompt'] == request['input']['params']['prompt'] + assert service.native_worker(job) is runtime['_run_generation'] + assert prepared_sfx_execution(job, job['params'], registry=native.registry('sound-output'), + check_models=lambda variant: check_sfx_models(runtime, variant)) + + +def test_concurrent_retries_keep_one_claim_and_new_intent_creates_one_more(tmp_path): + native, service, _, _ = setup_service(tmp_path) + with ThreadPoolExecutor(max_workers=4) as executor: + replies = list(executor.map(lambda _: _run(service.submit(command())), range(8))) + assert all(reply['receipt'] == replies[0]['receipt'] for reply in replies) + assert len(native.dispatch_calls) == 1 + _run(service.submit(command('another-intent'))) + assert len(native.dispatch_calls) == 2 + changed = command() + changed['input']['params']['duration_seconds'] = 4 + with pytest.raises(HTTPException) as conflict: + _run(service.submit(changed)) + assert conflict.value.status_code == 409 + + +def test_video_scope_and_inspected_duration_survive_admission(tmp_path, monkeypatch): + native, service, runtime, _ = setup_service(tmp_path) + source = tmp_path / 'source' / 'guide.mp4' + source.parent.mkdir() + source.write_bytes(b'inspected source video') + monkeypatch.setattr('services.studio_sfx_resources.probe_media', lambda _path: { + 'duration': 6.25, 'width': 640, 'height': 360, + }) + request = command() + request['input']['params']['video_guide'] = '/api/v1/file/guide.mp4?workspace=source' + _run(service.submit(request)) + job = native.dispatch_calls[0] + assert job['params']['video_guide'] == str(source) + assert job['params']['duration_seconds'] == 6.25 + record = native.registry('sound-output').command_admission(request['intent_id']) + assert record['original'] == request + assert record['effective']['resources'][0]['workspace'] == 'source' + assert prepared_sfx_execution(job, job['params'], registry=native.registry('sound-output'), + check_models=lambda variant: check_sfx_models(runtime, variant)) + source.unlink() + with pytest.raises(ValueError, match='no longer available'): + prepared_sfx_execution(job, job['params'], registry=native.registry('sound-output'), + check_models=lambda variant: check_sfx_models(runtime, variant)) + + +def test_missing_dependency_prevents_admission_and_removed_dependency_stops_worker(tmp_path): + native, service, runtime, weights = setup_service(tmp_path) + first = _run(service.submit(command())) + (weights / required_mmaudio_files('v2')[0]).unlink() + replay = _run(service.submit(command())) + assert replay == {"receipt": first['receipt'], "replayed": True} + with pytest.raises(HTTPException) as missing: + _run(service.submit(command('new-request'))) + assert missing.value.status_code == 409 + assert native.registry('sound-output').command_admission('new-request') is None + job = native.dispatch_calls[0] + with pytest.raises(ValueError, match='not installed'): + prepared_sfx_execution(job, job['params'], registry=native.registry('sound-output'), + check_models=lambda variant: check_sfx_models(runtime, variant)) + + +def test_provenance_alone_cannot_select_the_sfx_worker(tmp_path): + native, service, _, _ = setup_service(tmp_path) + _run(service.submit(command())) + forged = deepcopy(native.dispatch_calls[0]) + forged['id'] = 'unadmitted' + with pytest.raises(HTTPException) as mismatch: + service.native_worker(forged) + assert mismatch.value.status_code == 503 diff --git a/tests/test_studio_sfx_execution.py b/tests/test_studio_sfx_execution.py new file mode 100644 index 000000000..2a3346add --- /dev/null +++ b/tests/test_studio_sfx_execution.py @@ -0,0 +1,74 @@ +"""An SFX command must not silently change sources while waiting in the queue.""" +from copy import deepcopy +from types import SimpleNamespace + +import pytest + +from services.studio_image_resources import file_identity +from services.studio_sfx_execution import prepared_sfx_execution + + +def admitted(tmp_path, *, video=True): + path = tmp_path / "guide.mp4" + path.write_bytes(b"inspected video bytes") + params = {"_mmaudio_variant": "v2", "duration_seconds": 6.25, "guidance_scale": 1, + "video_guide": str(path) if video else None, "prompt": " rain\n "} + job = {"id": "job-1", "task_id": "task-generation-job-1", "workspace": "sfx-out", + "provenance": {"capability": "generation.sfx", "command": {"command_id": "intent-1"}}} + resources = [{"role": "video_guide", **file_identity(path)}] if video else [] + entry = {"operation": "generation.sfx", "task_id": job["task_id"], + "effective": {"resources": resources, "runtime": { + "params": deepcopy(params), "workspace": job["workspace"]}}} + task = {"id": job["task_id"], "backend_job_id": job["id"], "workspace": job["workspace"]} + registry = SimpleNamespace(command_admission=lambda intent: entry if intent == "intent-1" else None, + get=lambda task_id: task if task_id == task["id"] else None) + checked = [] + return path, params, job, registry, checked + + +@pytest.mark.parametrize("video", [False, True]) +def test_verified_native_request_keeps_its_exact_duration_and_checks_installed_models(tmp_path, video): + _, params, job, registry, checked = admitted(tmp_path, video=video) + before = deepcopy(params) + assert prepared_sfx_execution(job, params, registry=registry, check_models=checked.append) + assert params == before + assert checked == ["v2"] + + +@pytest.mark.parametrize("change", ["missing", "replacement", "different_task", "different_intent", "duration", "remove_guide", "boolean_as_number"]) +def test_changed_queued_request_stops_before_model_work(tmp_path, change): + path, params, job, registry, checked = admitted(tmp_path) + if change == "missing": + path.unlink() + elif change == "replacement": + path.write_bytes(b"replaced video bytes!") + elif change == "different_task": + job["task_id"] = "another-task" + elif change == "different_intent": + job["provenance"]["command"]["command_id"] = "unadmitted" + elif change == "duration": + params["duration_seconds"] = 20 + elif change == "remove_guide": + params["video_guide"] = None + elif change == "boolean_as_number": + params["guidance_scale"] = True + with pytest.raises(ValueError): + prepared_sfx_execution(job, params, registry=registry, check_models=checked.append) + assert checked == [] + + +def test_legacy_provenance_does_not_grant_typed_execution(tmp_path): + _, params, job, registry, checked = admitted(tmp_path) + job["provenance"] = {"capability": "generate"} + assert not prepared_sfx_execution(job, params, registry=registry, check_models=checked.append) + assert checked == [] + + +def test_missing_model_after_admission_fails_without_download_fallback(tmp_path): + _, params, job, registry, _ = admitted(tmp_path) + + def unavailable(_variant): + raise FileNotFoundError("The installed model was removed") + + with pytest.raises(FileNotFoundError, match="model was removed"): + prepared_sfx_execution(job, params, registry=registry, check_models=unavailable) diff --git a/tests/test_studio_sfx_native_worker.py b/tests/test_studio_sfx_native_worker.py new file mode 100644 index 000000000..13a2475a8 --- /dev/null +++ b/tests/test_studio_sfx_native_worker.py @@ -0,0 +1,100 @@ +"""Exercise the actual worker with a recorded admission and a provider stand-in. + +No model is loaded and the stand-in bytes are not claimed to be decoded media. +""" +import ast +import copy +import os +from pathlib import Path +import sys +import time +import traceback +from types import SimpleNamespace + +import pytest + +from tests.test_studio_sfx_commands import command, setup_service +from tests.test_image_generation_commands import _run + + +@pytest.fixture(scope='module') +def worker_source(): + source = Path(__file__).parents[1] / 'app' / '_launch_runtime.py' + tree = ast.parse(source.read_text(encoding='utf-8')) + worker = next(node for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == '_run_sfx_generation') + return ast.Module(body=[worker], type_ignores=[]), str(source) + + +@pytest.mark.parametrize('guided', [False, True]) +def test_admitted_worker_uses_exact_inputs_without_download_or_reprobe( + tmp_path, monkeypatch, worker_source, guided, +): + native, service, runtime, _weights = setup_service(tmp_path) + request = command() + request['input']['params'].update({ + 'guidance_scale': 3.5, 'sfx_text_weight': 0.7, + 'MMAudio_neg_prompt': ' Speech\nMusic ', + }) + if guided: + source = tmp_path / 'source' / 'guide.mp4' + source.parent.mkdir() + source.write_bytes(b'inspected video fixture') + request['input']['params']['video_guide'] = '/api/v1/file/guide.mp4?workspace=source' + monkeypatch.setattr('services.studio_sfx_resources.probe_media', lambda _path: { + 'duration': 6.25, 'width': 640, 'height': 360, + }) + _run(service.submit(request)) + job = native.dispatch_calls[0] + job['out_dir'] = str(tmp_path / 'native-output') + raw_params = copy.deepcopy(job['params']) + calls, sidecars, completed, probes = [], [], [], [] + + def generate(**kwargs): + calls.append(kwargs) + Path(kwargs['save_path']).write_bytes(b'provider stand-in; not real media') + + def finish(_job, status, **kwargs): + completed.append((status, kwargs)) + return status == 'completed' + + def forbidden_download(**_kwargs): + pytest.fail('An admitted SFX command must never provision models') + + wgp = runtime['wgp'] + wgp.server_config = {} + wgp.save_path = str(tmp_path / 'unused') + wgp.MMAUDIO_PERSIST_RAM = 'ram' + wgp.get_mmaudio_settings = lambda *_args, **_kwargs: (True, None, 'none', 'large_44k_v2', 'weights') + wgp.download_mmaudio = forbidden_download + wgp.get_available_filename = lambda directory, name, **_kwargs: os.path.join(directory, name) + wgp.format_time = str + monkeypatch.setitem(sys.modules, 'postprocessing.mmaudio.mmaudio', SimpleNamespace(video_to_audio=generate)) + monkeypatch.setitem(sys.modules, 'decord', SimpleNamespace(VideoReader=lambda path: probes.append(path))) + scope = { + 'os': os, 'time': time, 'copy': copy, 'traceback': traceback, 'wgp': wgp, + '_task_registry': native.registry, + 'is_cancel_requested': lambda _job: False, + 'update_job': lambda *_args, **_kwargs: True, + 'finish_job': finish, + 'record_job_outputs': lambda target, files: target.update(outputs=files), + '_publish_generation_sidecar_for_studio_job': lambda _job, path, metadata: sidecars.append((path, copy.deepcopy(metadata))), + } + module, filename = worker_source + exec(compile(module, filename, 'exec'), scope) + assert scope['_run_sfx_generation'](job, raw_params, time.time()) is True + assert len(calls) == 1 + call = calls[0] + assert call['prompt'] == request['input']['params']['prompt'] + assert call['negative_prompt'] == ' Speech\nMusic ' + assert call['duration'] == (6.25 if guided else 3) + assert call['audio_file_only'] is not guided + assert call['video'] == (str(source) if guided else None) + assert (call['num_steps'], call['cfg_strength'], call['text_weight'], call['seed']) == (25, 3.5, 0.7, 42) + assert probes == [] + assert len(sidecars) == 1 + output, metadata = sidecars[0] + assert output.endswith('.mp4' if guided else '.wav') + assert metadata['params'] == raw_params == job['params'] + assert metadata['upload_filenames'] == ({'video_guide': 'guide.mp4'} if guided else {}) + assert completed[0][0] == 'completed' diff --git a/tests/test_studio_sfx_preparation.py b/tests/test_studio_sfx_preparation.py new file mode 100644 index 000000000..19ebcfedf --- /dev/null +++ b/tests/test_studio_sfx_preparation.py @@ -0,0 +1,156 @@ +"""Pure preflight tests for the typed Studio SFX adapter.""" + +from copy import deepcopy + +import pytest +from fastapi import HTTPException + +from services.studio_sfx_preparation import prepare_studio_sfx + + +def params(**overrides): + result = { + "workspace": "sfx-output", + "model_type": "mmaudio_v2", + "prompt": " rain\nwith thunder ", + "MMAudio_neg_prompt": "speech", + "duration_seconds": 5.0, + "video_guide": None, + } + result.update(overrides) + return result + + +class FakeResources: + def __init__(self, result=None, error=None): + self.result = result + self.error = error + self.calls = [] + + def prepare_media(self, incoming): + self.calls.append(deepcopy(incoming)) + if self.error: + raise self.error + return deepcopy(self.result or (incoming, [])) + + +def invoke(incoming, *, resources=None, downloaded=True, definition=None, + policy_error=None): + resources = resources or FakeResources() + calls = {"policy": [], "definition": [], "downloaded": []} + + def policy(workspace): + calls["policy"].append(workspace) + if policy_error: + raise policy_error + + def model_definition(model_type): + calls["definition"].append(model_type) + return deepcopy(definition or {"architecture": "mmaudio"}) + + def model_downloaded(model_type): + calls["downloaded"].append(model_type) + return downloaded + + result = prepare_studio_sfx( + incoming, + model_definition=model_definition, + model_downloaded=model_downloaded, + resources=resources, + execution_policy=policy, + ) + return result, calls, resources + + +def test_text_preparation_is_detached_and_sets_native_mmaudio_markers(): + incoming = params() + before = deepcopy(incoming) + (native, identities), calls, resources = invoke(incoming) + assert incoming == before + assert native is not incoming + assert native["prompt"] == before["prompt"] + assert native["MMAudio_prompt"] == before["prompt"] + assert native["MMAudio_neg_prompt"] == "speech" + assert native["_mmaudio_variant"] == "v2" + assert native["MMAudio_setting"] == 1 + assert native["sfx_mode"] is True + assert native["duration_source"] == "text" + assert native["duration_seconds_requested"] == 5.0 + assert native["duration_seconds_effective"] == 5.0 + assert identities == [] + assert calls["policy"] == ["sfx-output"] + assert calls["definition"] == ["mmaudio_v2"] + assert calls["downloaded"] == ["mmaudio_v2"] + assert resources.calls[0] == before + + +def test_video_preparation_uses_inspected_duration_and_keeps_requested_control(): + media = FakeResources( + result=( + {**params(duration_seconds=60, video_guide="/api/v1/file/guide.mp4?workspace=source"), + "video_guide": "/srv/source/guide.mp4"}, + [{"role": "video_guide", "workspace": "source", "sha256": "guide-hash", + "size_bytes": 12, "duration_seconds": 31.25, "width": 640, "height": 360}], + ) + ) + (native, identities), _, _ = invoke( + params(duration_seconds=60, video_guide="/api/v1/file/guide.mp4?workspace=source"), + resources=media, + ) + assert native["video_guide"] == "/srv/source/guide.mp4" + assert native["duration_seconds"] == 31.25 + assert native["duration_seconds_requested"] == 60.0 + assert native["duration_seconds_effective"] == 31.25 + assert native["duration_source"] == "video" + assert identities[0]["workspace"] == "source" + assert identities[0]["sha256"] == "guide-hash" + + +def test_missing_selected_video_is_a_preflight_error_without_text_fallback(): + resources = FakeResources(error=ValueError("selected guide is missing")) + with pytest.raises(HTTPException) as error: + invoke(params(video_guide="/api/v1/file/missing.mp4?workspace=source"), resources=resources) + assert error.value.status_code == 422 + assert "missing" in error.value.detail["message"] + assert len(resources.calls) == 1 + + +@pytest.mark.parametrize( + ("definition", "downloaded", "status"), + [ + ({"architecture": "ltx2"}, True, 422), + ({"architecture": "mmaudio", "variant": "nsfw"}, True, 422), + ({"architecture": "mmaudio"}, False, 409), + ], +) +def test_model_definition_and_installed_state_are_checked_before_media( + definition, downloaded, status +): + resources = FakeResources() + with pytest.raises(HTTPException) as error: + invoke(params(), definition=definition, downloaded=downloaded, resources=resources) + assert error.value.status_code == status + assert resources.calls == [] + + +def test_policy_runs_before_model_or_resource_inspection(): + resources = FakeResources() + with pytest.raises(HTTPException) as error: + invoke(params(), resources=resources, policy_error=HTTPException(403, "isolated workspace")) + assert error.value.status_code == 403 + assert resources.calls == [] + + +def test_model_downloaded_is_the_single_installed_dependency_gate(): + with pytest.raises(HTTPException) as error: + invoke(params(), downloaded=False) + assert error.value.status_code == 409 + + +@pytest.mark.parametrize("duration", [0, -1, float("nan"), float("inf")]) +def test_invalid_duration_fails_before_media(duration): + resources = FakeResources() + with pytest.raises(HTTPException) as error: + invoke(params(duration_seconds=duration), resources=resources) + assert error.value.status_code == 422 + assert resources.calls == [] diff --git a/tests/test_studio_sfx_resources.py b/tests/test_studio_sfx_resources.py new file mode 100644 index 000000000..31217e63d --- /dev/null +++ b/tests/test_studio_sfx_resources.py @@ -0,0 +1,146 @@ +"""Canonical video guide resolution and identity tests for Studio SFX.""" + +from copy import deepcopy +import hashlib + +import pytest + +from services.asset_manifest import build_asset_manifest, write_asset_manifest +from services.studio_sfx_resources import StudioSfxResources + + +@pytest.fixture +def resources(tmp_path, monkeypatch): + folders = { + name: tmp_path / name + for name in ("uploads", "source", "destination", "default") + } + for folder in folders.values(): + folder.mkdir() + monkeypatch.setattr( + "services.studio_sfx_resources.probe_media", + lambda _path: { + "duration": 31.25, + "width": 640, + "height": 360, + "fps": 25.0, + "has_audio": False, + "pixel_format": "yuv420p", + "has_alpha": False, + }, + ) + resolver = StudioSfxResources( + workspace_dir=lambda name: folders[name], + uploads_dir=lambda: folders["uploads"], + list_workspaces=lambda: [{"name": name} for name in ("source", "destination", "default")], + lora_search_dirs=lambda _model: [], + lora_compatible=lambda *_args: False, + ) + return folders, resolver + + +def write_video(path, content=b"video-guide"): + path.write_bytes(content) + + +def test_video_url_resolves_against_declared_source_and_returns_portable_identity(resources): + folders, resolver = resources + source = folders["source"] / "guide.mp4" + destination = folders["destination"] / "guide.mp4" + write_video(source, b"source-guide") + write_video(destination, b"different-destination-guide") + params = { + "workspace": "destination", + "video_guide": "/api/v1/file/guide.mp4?workspace=source", + } + before = deepcopy(params) + working, identities = resolver.prepare_media(params) + assert params == before + assert working["video_guide"] == str(source) + assert identities == [ + { + "role": "video_guide", + "index": 0, + "url": "/api/v1/file/guide.mp4?workspace=source", + "workspace": "source", + "duration_seconds": 31.25, + "width": 640, + "height": 360, + "fps": 25.0, + "has_audio": False, + "pixel_format": "yuv420p", + "has_alpha": False, + "sha256": hashlib.sha256(b"source-guide").hexdigest(), + "size_bytes": len(b"source-guide"), + } + ] + assert not any("path" in key for key in identities[0]) + + +def test_text_only_request_has_no_resource_and_no_path(resources): + folders, resolver = resources + params = {"workspace": "destination", "video_guide": None} + working, identities = resolver.prepare_media(params) + assert working["video_guide"] is None + assert identities == [] + + +def test_missing_declared_source_does_not_fallback_to_destination(resources): + folders, resolver = resources + write_video(folders["destination"] / "guide.mp4") + with pytest.raises(ValueError, match="not available|missing"): + resolver.prepare_media({ + "workspace": "destination", + "video_guide": "/api/v1/file/guide.mp4?workspace=source", + }) + + +def test_selected_non_video_is_rejected_after_canonical_resolution(resources, monkeypatch): + folders, resolver = resources + write_video(folders["source"] / "guide.mp4") + monkeypatch.setattr( + "services.studio_sfx_resources.probe_media", + lambda _path: (_ for _ in ()).throw(ValueError("no video stream")), + ) + with pytest.raises(ValueError, match="readable video"): + resolver.prepare_media({ + "video_guide": "/api/v1/file/guide.mp4?workspace=source", + }) + + +def test_probe_requires_positive_finite_duration_and_dimensions(resources, monkeypatch): + folders, resolver = resources + write_video(folders["source"] / "guide.mp4") + for metadata, message in ( + ({"duration": 0, "width": 640, "height": 360}, "duration"), + ({"duration": float("nan"), "width": 640, "height": 360}, "duration"), + ({"duration": 1, "width": 0, "height": 360}, "dimensions"), + ): + monkeypatch.setattr( + "services.studio_sfx_resources.probe_media", + lambda _path, metadata=metadata: metadata, + ) + with pytest.raises(ValueError, match=message): + resolver.prepare_media({"video_guide": "/api/v1/file/guide.mp4?workspace=source"}) + + +def test_video_asset_id_requires_one_unique_location(resources): + folders, resolver = resources + source = folders["source"] / "guide.mp4" + write_video(source) + write_asset_manifest(source, build_asset_manifest(source, asset_id="asset_guide", kind="video")) + working, identities = resolver.prepare_media({"video_guide": "asset_guide"}) + assert working["video_guide"] == str(source) + assert identities[0]["url"] == "asset_guide" + second = folders["destination"] / "guide.mp4" + write_video(second, b"different") + write_asset_manifest(second, build_asset_manifest(second, asset_id="asset_guide", kind="video")) + with pytest.raises(ValueError, match="multiple locations"): + resolver.prepare_media({"video_guide": "asset_guide"}) + + +def test_upload_reference_cannot_override_source_workspace(resources): + folders, resolver = resources + write_video(folders["uploads"] / "guide.mp4") + with pytest.raises(ValueError, match="override"): + resolver.prepare_media({"video_guide": "/api/v1/uploads/guide.mp4?workspace=source"}) diff --git a/tests/test_studio_sfx_spec.py b/tests/test_studio_sfx_spec.py new file mode 100644 index 000000000..c018088ad --- /dev/null +++ b/tests/test_studio_sfx_spec.py @@ -0,0 +1,188 @@ +"""Adversarial tests for the closed Studio SFX command envelope.""" + +from copy import deepcopy +import hashlib +import json + +import pytest + +from services.studio_sfx_spec import ( + StudioSfxSpecError, + freeze_studio_sfx_spec, + studio_sfx_schema, +) + + +def command(**overrides): + params = { + "model_type": "mmaudio_v2", + "prompt": " rain on a tin roof\nwith distant thunder ", + "duration_seconds": 7, + } + params.update(overrides) + return { + "version": 2, + "operation": "generation.sfx", + "intent_id": "sfx-intent-1", + "input": {"workspace": "sfx-output", "params": params}, + } + + +def test_freeze_preserves_literal_original_and_derives_native_selectors(): + submitted = command(MMAudio_neg_prompt="speech, singing", seed=20260909) + before = deepcopy(submitted) + frozen = freeze_studio_sfx_spec(submitted) + assert submitted == before + assert frozen["original"] == before + assert frozen["original"] is not submitted + effective = frozen["effective"]["input"]["params"] + assert effective["prompt"] == before["input"]["params"]["prompt"] + assert effective["MMAudio_prompt"] == before["input"]["params"]["prompt"] + assert effective["MMAudio_neg_prompt"] == "speech, singing" + assert effective["_mmaudio_variant"] == "v2" + assert effective["generation_mode"] == "audio" + assert effective["_audio_sub_mode"] == "sfx" + assert effective["MMAudio_setting"] == 1 + assert effective["sfx_mode"] is True + assert effective["duration_seconds_requested"] == 7.0 + assert effective["duration_seconds_effective"] is None + assert effective["duration_source"] == "text" + + +def test_mmaudio_prompt_spelling_is_preserved_and_mismatch_fails_closed(): + frozen = freeze_studio_sfx_spec(command(prompt=None, MMAudio_prompt=" exact\ntext ")) + assert frozen["original"]["input"]["params"]["MMAudio_prompt"] == " exact\ntext " + assert frozen["effective"]["input"]["params"]["prompt"] == " exact\ntext " + + with pytest.raises(StudioSfxSpecError, match="must match"): + freeze_studio_sfx_spec(command(MMAudio_prompt="different")) + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("generation_mode", "video"), + ("_audio_sub_mode", "music"), + ("image_mode", 1), + ("video_length", 1), + ("MMAudio_setting", 0), + ("sfx_mode", False), + ("_mmaudio_variant", "nsfw"), + ("model_type", "carrier_video_model"), + ("num_inference_steps", 24), + ("guidance_scale", True), + ("sfx_text_weight", float("nan")), + ], +) +def test_incompatible_selectors_and_authority_markers_are_rejected(field, value): + with pytest.raises(StudioSfxSpecError): + freeze_studio_sfx_spec(command(**{field: value})) + + +@pytest.mark.parametrize( + "value", + [ + "/etc/passwd", + "../guide.mp4", + "C:\\guide.mp4", + "https://example.test/guide.mp4", + "/api/v1/file/guide.mp4", + "/api/v1/file/../guide.mp4?workspace=source", + "/api/v1/uploads/guide.mp4?workspace=source", + ], +) +def test_video_guide_accepts_only_canonical_local_references(value): + with pytest.raises(StudioSfxSpecError): + freeze_studio_sfx_spec(command(video_guide=value)) + + +@pytest.mark.parametrize("value", [None, "", "asset_video_123", "/api/v1/uploads/guide.mp4"]) +def test_video_guide_sentinels_and_asset_references_remain_literal(value): + frozen = freeze_studio_sfx_spec(command(video_guide=value)) + assert frozen["original"]["input"]["params"]["video_guide"] == value + assert frozen["effective"]["input"]["params"]["video_guide"] == value + + +def test_text_only_duration_is_limited_but_video_control_is_preserved_for_preparation(): + with pytest.raises(StudioSfxSpecError, match="at most 20"): + freeze_studio_sfx_spec(command(duration_seconds=20.01)) + frozen = freeze_studio_sfx_spec( + command(duration_seconds=60, video_guide="/api/v1/file/long.mp4?workspace=source") + ) + params = frozen["effective"]["input"]["params"] + assert params["duration_seconds"] == 60.0 + assert params["duration_seconds_requested"] == 60.0 + assert params["duration_source"] == "video" + assert params["duration_seconds_effective"] is None + + +def test_fingerprint_is_stable_and_excludes_intent_id(): + first = freeze_studio_sfx_spec(command()) + second_command = command() + second_command["intent_id"] = "sfx-intent-2" + second = freeze_studio_sfx_spec(second_command) + assert first["fingerprint"] == second["fingerprint"] + content = { + "version": 2, + "operation": "generation.sfx", + "input": first["effective"]["input"], + } + expected = hashlib.sha256( + json.dumps(content, ensure_ascii=False, sort_keys=True, + separators=(",", ":"), allow_nan=False).encode() + ).hexdigest() + assert first["fingerprint"] == expected + + +def test_collection_id_is_transport_metadata_and_is_fingerprinted(): + first = freeze_studio_sfx_spec(command()) + with_collection = command() + with_collection["input"] = { + **with_collection["input"], + "workspace_collection_id": "collection-sfx-1", + } + second = freeze_studio_sfx_spec(with_collection) + assert second["effective"]["input"]["workspace_collection_id"] == "collection-sfx-1" + assert first["fingerprint"] != second["fingerprint"] + + +def test_unknown_provider_and_host_authority_fields_do_not_cross_closed_boundary(): + for field, value in ( + ("provider_payload", {"duration": 2}), + ("provenance", {"actor": "wizard"}), + ("actor", "wizard"), + ("video_carrier_model", "ltx2"), + ("download", True), + ("workspace", "/tmp/private"), + ): + invalid = command() + if field == "workspace": + invalid["input"]["workspace"] = value + else: + invalid["input"]["params"][field] = value + with pytest.raises(StudioSfxSpecError): + freeze_studio_sfx_spec(invalid) + + +def test_schema_publishes_video_derived_duration_and_no_host_paths(): + schema = studio_sfx_schema() + assert schema["version"] == 2 + assert schema["operation"] == "generation.sfx" + params_schema = schema["input"]["$defs"]["StudioSfxParams"] + assert params_schema["additionalProperties"] is False + assert "video_guide" in params_schema["properties"] + assert "maximum" not in params_schema["properties"]["duration_seconds"] + assert schema["limits"]["text_duration_seconds"]["maximum"] == 20 + assert "video_duration_seconds" in schema["limits"] + assert "filesystem paths" in schema["excluded"] + + +@pytest.mark.parametrize("blank", ["", " ", "\n"]) +@pytest.mark.parametrize("field", ["prompt", "MMAudio_prompt"]) +def test_explicit_blank_alias_cannot_override_the_other_literal(field, blank): + submitted = command(prompt="literal sound", MMAudio_prompt="literal sound") + submitted["input"]["params"][field] = blank + before = deepcopy(submitted) + with pytest.raises(StudioSfxSpecError, match="must match"): + freeze_studio_sfx_spec(submitted) + assert submitted == before diff --git a/tests/test_studio_speech_commands.py b/tests/test_studio_speech_commands.py new file mode 100644 index 000000000..d02c279ef --- /dev/null +++ b/tests/test_studio_speech_commands.py @@ -0,0 +1,149 @@ +"""Shared speech admission, native snapshots and inter-client replay without a provider.""" +from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy +import json +from pathlib import Path + +import pytest +from fastapi import FastAPI, HTTPException +from fastapi.testclient import TestClient + +from routers.image_generation_commands import create_image_generation_commands_router, image_command_handlers +from routers.studio_speech_commands import speech_command_catalog +from services.native_generation_operation import NativeGenerationOperation +from services.studio_speech_spec import freeze_studio_speech_spec, studio_speech_schema +from tests.test_image_generation_commands import FakeNative, _command as image_command, _run, _db_counts + + +def speech_command(intent="speech-test-intent"): + native = json.loads((Path(__file__).parent / "fixtures/studio_speech_native_request.json").read_text()) + workspace = native.pop("workspace") + return {"version": 2, "operation": "generation.speech", "intent_id": intent, + "input": {"workspace": workspace, "params": native}} + + +def configured_service(native): + def freeze(command): + frozen = freeze_studio_speech_spec(command) + effective = frozen["effective"]["input"] + return frozen, {**deepcopy(effective["params"]), "workspace": effective["workspace"]} + + def prepare(params): + native.preflight(params) + return deepcopy(params), [] + + service = native.service() + service.operations = {"generation.speech": NativeGenerationOperation( + freeze=freeze, prepare=prepare, catalog=speech_command_catalog())} + return service + + +def test_http_then_mcp_replays_same_speech_task_and_native_literal_snapshot(tmp_path): + native = FakeNative(tmp_path) + service = configured_service(native) + app = FastAPI() + app.include_router(create_image_generation_commands_router(service)) + command = speech_command() + with TestClient(app) as client: + first = client.post("/api/v1/generation/commands", json=command, + headers={"X-Hocus-UI-Surface": "wizard"}) + assert first.status_code == 200 + catalog = client.get("/api/v1/generation/commands").json() + names = {operation["name"] for operation in catalog["operations"]} + assert {"generation.image", "generation.speech", "generation.receipt"} <= names + mcp_arguments = {key: value for key, value in command.items() if key != "operation"} + replay = _run(image_command_handlers(service)["generation.speech"](mcp_arguments)) + receipt = first.json()["receipt"] + assert replay == {"receipt": receipt, "replayed": True} + assert receipt["operation"] == "generation.speech" + assert native.prepare_calls == native.preflight_calls == 1 + assert len(native.dispatch_calls) == 1 + entry = native.registry("speech-test").command_admission(command["intent_id"]) + snapshot = entry["effective"]["runtime"] + assert snapshot["params"]["prompt"] == command["input"]["params"]["prompt"] + assert snapshot["params"]["_tts_original_prompt"] == command["input"]["params"]["_tts_original_prompt"] + assert snapshot["params"]["duration_seconds"] == 20 + assert snapshot["provenance"]["capability"] == "generation.speech" + assert snapshot["provenance"]["actor"] == "wizard" + + +def test_intent_cannot_cross_image_and_speech_domains(tmp_path): + native = FakeNative(tmp_path) + service = configured_service(native) + _run(service.submit(speech_command())) + with pytest.raises(HTTPException) as caught: + _run(service.submit(image_command("speech-test-intent", workspace="speech-test"))) + assert caught.value.status_code == 409 + assert len(native.dispatch_calls) == 1 + + +def test_concurrent_speech_clients_and_deliberate_second_attempt(tmp_path): + native = FakeNative(tmp_path) + service = configured_service(native) + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(lambda _: _run(service.submit(speech_command())), range(2))) + assert results[0]["receipt"] == results[1]["receipt"] + assert sum(not result["replayed"] for result in results) == 1 + assert len(native.dispatch_calls) == 1 + second = _run(service.submit(speech_command("another-deliberate-speech"))) + assert second["receipt"]["taskIds"] != results[0]["receipt"]["taskIds"] + assert _db_counts(native.registry("speech-test"))["tasks"] == 2 + + +def test_speech_replay_does_not_revalidate_an_unavailable_model(tmp_path): + native = FakeNative(tmp_path) + service = configured_service(native) + first = _run(service.submit(speech_command())) + native.preflight_error = RuntimeError("Model is now unavailable") + assert _run(service.submit(speech_command()))["receipt"] == first["receipt"] + changed = speech_command() + changed["input"]["params"]["prompt"] += " Changed" + with pytest.raises(HTTPException) as caught: + _run(service.submit(changed)) + assert caught.value.status_code == 409 + assert len(native.dispatch_calls) == 1 + + +def test_speech_restart_rebuilds_existing_recovery_without_dispatch(tmp_path): + first_native = FakeNative(tmp_path) + first = _run(configured_service(first_native).submit(speech_command())) + restarted = FakeNative(tmp_path, interrupt_stale=True) + service = configured_service(restarted) + service.restore_recovery(["speech-test"]) + assert len(restarted.persist_calls) == 1 + assert restarted.dispatch_calls == [] + record = restarted.persist_calls[0] + assert record["params"]["_tts_original_prompt"] == speech_command()["input"]["params"]["prompt"] + assert service.filter_recovery([record]) == [record] + replay = _run(service.submit(speech_command())) + assert replay["receipt"] == first["receipt"] + assert service.receipt("speech-test", "speech-test-intent")["task"]["status"] == "interrupted" + + +def test_runtime_without_speech_adapter_does_not_adopt_its_recovery(tmp_path): + _run(configured_service(FakeNative(tmp_path)).submit(speech_command())) + restarted = FakeNative(tmp_path, interrupt_stale=True) + restarted.service().restore_recovery(["speech-test"]) + assert restarted.persist_calls == [] + assert restarted.dispatch_calls == [] + + +def test_speech_orphan_and_malformed_command_do_not_block_linked_recovery(tmp_path): + _run(configured_service(FakeNative(tmp_path)).submit(speech_command())) + restarted = FakeNative(tmp_path, interrupt_stale=True) + service = configured_service(restarted) + service.restore_recovery(["speech-test"]) + linked = restarted.persist_calls[0] + orphan = deepcopy(linked) + orphan["provenance"]["command"]["command_id"] = "missing-speech-admission" + malformed = deepcopy(linked) + malformed["provenance"]["command"] = "invalid" + assert service.filter_recovery([orphan, malformed, linked]) == [linked] + service.discard_recovery([orphan, malformed, linked]) + assert service.receipt("speech-test", "speech-test-intent")["task"]["status"] == "cancelled" + + +def test_browser_catalog_matches_executable_speech_contract(): + path = Path(__file__).resolve().parents[1] / "ui/src/api/speechCommandCatalog.json" + projection = json.loads(path.read_text(encoding="utf-8")) + assert projection == {"version": 2, "operations": [speech_command_catalog()], "studio": studio_speech_schema()} diff --git a/tests/test_studio_speech_preparation.py b/tests/test_studio_speech_preparation.py new file mode 100644 index 000000000..19e8d63e2 --- /dev/null +++ b/tests/test_studio_speech_preparation.py @@ -0,0 +1,349 @@ +"""Provider-free tests for Studio speech model/resource preparation.""" + +from copy import deepcopy +import json +from pathlib import Path + +import pytest +from fastapi import HTTPException + +from services.studio_speech_preparation import prepare_studio_speech + + +FIXTURE = Path(__file__).parent / "fixtures" / "studio_speech_native_request.json" + +KUGEL_DEFINITION = { + "audio_only": True, + "image_outputs": False, + "guidance_max_phases": 1, + "no_negative_prompt": True, + "inference_steps": False, + "temperature": True, + "duration_slider": {"min": 1, "max": 600, "default": 20}, + "audio_prompt_type_sources": {"selection": ["", "A", "AB"], "default": ""}, + "audio_mode_from_voice_count": True, + "max_voice_count": 6, + "custom_settings": [{"id": "auto_split_every_s", "type": "float", "min": 5, "max": 90}], +} + + +def base_params(**overrides): + params = json.loads(FIXTURE.read_text()) + params.update(overrides) + return params + + +class FakeResources: + def __init__(self, *, media_result=None, lora_result=None, media_error=None): + self.media_result = media_result + self.lora_result = lora_result if lora_result is not None else [] + self.media_error = media_error + self.media_calls = [] + self.lora_calls = [] + + def prepare_media(self, params): + self.media_calls.append(deepcopy(params)) + if self.media_error is not None: + raise self.media_error + return deepcopy(self.media_result if self.media_result is not None else (params, [])) + + def prepare_loras(self, params, definition): + self.lora_calls.append((deepcopy(params), deepcopy(definition))) + return deepcopy(self.lora_result) + + +def invoke( + params, + *, + definition=None, + downloaded=True, + resources=None, + policy_error=None, +): + definition = deepcopy(definition or KUGEL_DEFINITION) + resources = resources or FakeResources() + policy_calls = [] + definition_calls = [] + download_calls = [] + + def model_definition(model_type): + definition_calls.append(model_type) + return deepcopy(definition) + + def model_downloaded(model_type): + download_calls.append(model_type) + return downloaded + + def policy(workspace): + policy_calls.append(workspace) + if policy_error is not None: + raise policy_error + + result = prepare_studio_speech( + params, + model_definition=model_definition, + model_downloaded=model_downloaded, + resources=resources, + execution_policy=policy, + ) + return result, { + "resources": resources, + "policy_calls": policy_calls, + "definition_calls": definition_calls, + "download_calls": download_calls, + } + + +def assert_http_error(error_info, *, status=422): + assert error_info.value.status_code == status + assert error_info.value.detail["code"] == ( + "model_unavailable" if status == 409 else "invalid_studio_speech_input" + ) + return error_info.value.detail["message"] + + +def test_installed_speech_model_returns_detached_native_snapshot_and_resources(): + params = base_params() + before = deepcopy(params) + resources = FakeResources( + media_result=( + {**params, "duration_seconds": 20.0, "nested": {"keep": [1]}}, + [{"role": "audio_guide", "sha256": "audio-hash"}], + ), + lora_result=[{"role": "lora", "name": "voice-style", "sha256": "lora-hash"}], + ) + + (native, identities), calls = invoke(params, resources=resources) + + assert params == before + assert native == resources.media_result[0] + assert native is not resources.media_result[0] + assert identities == [ + {"role": "audio_guide", "sha256": "audio-hash"}, + {"role": "lora", "name": "voice-style", "sha256": "lora-hash"}, + ] + assert calls["policy_calls"] == ["speech-test"] + assert calls["definition_calls"] == ["kugelaudio_0_open"] + assert calls["download_calls"] == ["kugelaudio_0_open"] + assert resources.media_calls[0] == before + assert resources.lora_calls[0][0] == before + native["nested"]["keep"].append(2) + assert resources.media_result[0]["nested"] == {"keep": [1]} + assert params == before + + +@pytest.mark.parametrize("model_type", ["minimax_music3", "ace_step_v1", "yue", "mmaudio_v2", "unknown_audio"]) +def test_music_sfx_and_unknown_audio_models_are_not_speech(model_type): + resources = FakeResources() + params = base_params(model_type=model_type) + with pytest.raises(HTTPException) as error: + invoke(params, resources=resources) + message = assert_http_error(error) + assert "speech" in message.lower() or "music" in message.lower() + assert resources.media_calls == resources.lora_calls == [] + + +@pytest.mark.parametrize( + ("definition", "downloaded", "status"), + [ + ({**KUGEL_DEFINITION, "audio_only": False}, True, 422), + ({**KUGEL_DEFINITION, "image_outputs": True}, True, 422), + (KUGEL_DEFINITION, False, 409), + ], +) +def test_model_must_be_installed_audio_only_and_downloaded(definition, downloaded, status): + resources = FakeResources() + with pytest.raises(HTTPException) as error: + invoke(base_params(), definition=definition, downloaded=downloaded, resources=resources) + assert_http_error(error, status=status) + assert resources.media_calls == resources.lora_calls == [] + + +def test_model_mode_default_is_declared_by_model_and_invalid_mode_fails_early(): + definition = { + **KUGEL_DEFINITION, + "model_modes": {"choices": [("English", "en"), ("Spanish", "es")], "default": "en"}, + "audio_mode_from_voice_count": False, + } + params = base_params() + (native, _), _ = invoke(params, definition=definition) + assert native["model_mode"] == "en" + + bad = base_params(model_mode="de") + with pytest.raises(HTTPException) as error: + invoke(bad, definition=definition) + assert "model_mode" in assert_http_error(error) + + +def test_duration_zero_is_preserved_when_the_selected_model_declares_auto_duration(): + definition = { + **KUGEL_DEFINITION, + "audio_mode_from_voice_count": False, + "inference_steps": True, + "guidance_max_phases": 0, + "temperature": False, + "duration_slider": {"min": 0, "max": 60, "default": 0}, + "audio_prompt_type_sources": {"selection": ["", "A", "AB"], "default": ""}, + } + params = base_params( + model_type="dramabox_audio", + duration_seconds=0, + guidance_phases=0, + ) + params.pop("temperature") + (native, _), _ = invoke(params, definition=definition) + assert native["duration_seconds"] == 0 + + +def test_locked_scenema_step_count_is_preserved_even_without_an_editable_step_control(): + definition = { + **KUGEL_DEFINITION, + "audio_mode_from_voice_count": False, + "inference_steps": False, + "lock_inference_steps": True, + "guidance_max_phases": 0, + "temperature": False, + "duration_slider": {"min": 1, "max": 1800, "default": 120}, + "audio_prompt_type_sources": {"selection": ["", "A2", "AB2"], "default": ""}, + } + params = base_params(guidance_phases=0, num_inference_steps=8, temperature=None) + (native, _), _ = invoke(params, definition=definition) + assert native["num_inference_steps"] == 8 + + +@pytest.mark.parametrize( + ("params", "definition", "needle"), + [ + (base_params(negative_prompt="forbidden"), KUGEL_DEFINITION, "negative_prompt"), + (base_params(num_inference_steps=2), KUGEL_DEFINITION, "num_inference_steps"), + (base_params(guidance_phases=2), KUGEL_DEFINITION, "guidance_phases"), + ( + base_params(temperature=1), + {**KUGEL_DEFINITION, "temperature": False}, + "temperature", + ), + ( + base_params(top_k=10), + KUGEL_DEFINITION, + "top_k", + ), + ], +) +def test_declared_native_capabilities_fail_before_resource_inspection(params, definition, needle): + resources = FakeResources() + with pytest.raises(HTTPException) as error: + invoke(params, definition=definition, resources=resources) + assert needle in assert_http_error(error) + assert resources.media_calls == resources.lora_calls == [] + + +def test_audio_prompt_selector_and_references_must_be_coherent(): + missing = base_params(audio_prompt_type="A", _tts_voice_count=1) + with pytest.raises(HTTPException) as error: + invoke(missing) + assert "audio_guide" in assert_http_error(error) + + orphan = base_params(audio_prompt_type="", _tts_voice_count=1, audio_guide="/api/v1/uploads/voice.wav") + with pytest.raises(HTTPException) as error: + invoke(orphan) + assert "audio_prompt_type" in assert_http_error(error) + + missing_second = base_params( + audio_prompt_type="AB", _tts_voice_count=2, + audio_guide="/api/v1/uploads/one.wav", + ) + with pytest.raises(HTTPException) as error: + invoke(missing_second) + assert "audio_guide2" in assert_http_error(error) + + valid = base_params( + audio_prompt_type="AB", _tts_voice_count=2, + audio_guide="/api/v1/uploads/one.wav", + audio_guide2="/api/v1/file/two.wav?workspace=speech-test", + ) + resources = FakeResources() + (native, _), _ = invoke(valid, resources=resources) + assert native["audio_guide"] == valid["audio_guide"] + assert native["audio_guide2"] == valid["audio_guide2"] + assert resources.media_calls[0]["audio_guide2"] == valid["audio_guide2"] + + +def test_voice_mode_modifiers_are_preserved_while_the_declared_base_mode_is_checked(): + definition = { + **KUGEL_DEFINITION, + "audio_prompt_type_sources": { + "selection": ["", "A2", "AB2"], + "default": "", + "custom_flags": {"2": "SeedVC"}, + }, + "audio_mode_from_voice_count": True, + "max_voice_count": 2, + } + params = base_params( + audio_prompt_type="AB2NV", + _tts_voice_count=2, + audio_guide="/api/v1/uploads/one.wav", + audio_guide2="/api/v1/uploads/two.wav", + ) + (native, _), _ = invoke(params, definition=definition) + assert native["audio_prompt_type"] == "AB2NV" + + +def test_voice_count_cannot_exceed_model_declared_limit(): + params = base_params(_tts_voice_count=3, audio_prompt_type="AB") + definition = {**KUGEL_DEFINITION, "max_voice_count": 2} + with pytest.raises(HTTPException) as error: + invoke(params, definition=definition) + assert "voice" in assert_http_error(error).lower() + + +def test_custom_settings_use_model_metadata_for_keys_types_and_ranges(): + bad_range = base_params(custom_settings={"auto_split_every_s": 4}) + with pytest.raises(HTTPException) as error: + invoke(bad_range) + assert "minimum" in assert_http_error(error) + + bad_unknown = base_params(custom_settings={"not_declared": 1}) + with pytest.raises(HTTPException) as error: + invoke(bad_unknown) + assert "unknown" in assert_http_error(error) + + valid = base_params(custom_settings={"auto_split_every_s": 5.0}) + (native, _), _ = invoke(valid) + assert native["custom_settings"] == valid["custom_settings"] + + +def test_loras_are_rejected_without_explicit_audio_lora_capability_and_resolved_when_enabled(): + params = base_params(activated_loras=["voice-style"]) + resources = FakeResources(lora_result=[{"role": "lora", "name": "voice-style"}]) + with pytest.raises(HTTPException) as error: + invoke(params, resources=resources) + assert "lora" in assert_http_error(error).lower() + assert resources.media_calls == resources.lora_calls == [] + + definition = {**KUGEL_DEFINITION, "enabled_audio_lora": True} + (native, identities), _ = invoke(params, definition=definition, resources=resources) + assert native["activated_loras"] == ["voice-style"] + assert identities == [{"role": "lora", "name": "voice-style"}] + assert len(resources.lora_calls) == 1 + + +def test_resource_failure_is_wrapped_and_loras_are_not_looked_up_after_media_failure(): + resources = FakeResources(media_error=ValueError("bad voice reference")) + with pytest.raises(HTTPException) as error: + invoke(base_params(), resources=resources) + assert "bad voice reference" in assert_http_error(error) + assert len(resources.media_calls) == 1 + assert resources.lora_calls == [] + + +def test_execution_policy_runs_before_model_lookup(): + resources = FakeResources() + with pytest.raises(HTTPException) as error: + invoke( + base_params(), + resources=resources, + policy_error=HTTPException(409, {"code": "execution_policy", "message": "busy"}), + ) + assert error.value.status_code == 409 + assert resources.media_calls == resources.lora_calls == [] diff --git a/tests/test_studio_speech_resources.py b/tests/test_studio_speech_resources.py new file mode 100644 index 000000000..3763125cc --- /dev/null +++ b/tests/test_studio_speech_resources.py @@ -0,0 +1,112 @@ +"""Canonical voice sources stay separate from the destination and native fallback.""" +from copy import deepcopy +import hashlib +import wave + +import pytest + +from services.studio_speech_resources import StudioSpeechResources +from services.asset_manifest import build_asset_manifest, write_asset_manifest +from services.wangp_submission import prepare_generation_inputs + + +def write_wave(path, frames=800): + with wave.open(str(path), "wb") as output: + output.setparams((1, 2, 8000, 0, "NONE", "not compressed")) + output.writeframes(b"\x01\x00" * frames) + + +@pytest.fixture +def sources(tmp_path): + folders = {name: tmp_path / name for name in ("uploads", "source", "destination", "default")} + for path in folders.values(): + path.mkdir() + resources = StudioSpeechResources( + workspace_dir=lambda name: folders[name], uploads_dir=lambda: folders["uploads"], + list_workspaces=lambda: [{"name": name} for name in ("source", "destination")], + lora_search_dirs=lambda _: [], lora_compatible=lambda *_: False, + ) + return folders, resources + + +@pytest.mark.parametrize("new_family", [False, True]) +def test_voice_references_use_declared_source_across_native_preparation(sources, new_family): + folders, resources = sources + write_wave(folders["source"] / "same.wav", 800) + write_wave(folders["destination"] / "same.wav", 1600) + write_wave(folders["uploads"] / "voice.wav", 2400) + params = {"generation_mode": "audio", "image_mode": 0, "workspace": "destination", + "audio_guide": "/api/v1/file/same.wav?workspace=source", + "audio_guide6": "/api/v1/uploads/voice.wav", "audio_guide2": ""} + original = deepcopy(params) + working, identities = resources.prepare_media(params) + prepare_generation_inputs(working, {"audio_only": True, "wangp_1272": new_family}, "destination", + workspace_dir=folders["destination"], uploads_dir=folders["uploads"], + prepared_speech=True) + assert params == original + assert working["audio_guide"] == str(folders["source"] / "same.wav") + assert working["audio_guide6"] == str(folders["uploads"] / "voice.wav") + assert working["audio_guide2"] == "" + assert [item["workspace"] for item in identities] == ["source", "__uploads__"] + assert identities[0]["duration_seconds"] == 0.1 + assert identities[0]["sha256"] == hashlib.sha256((folders["source"] / "same.wav").read_bytes()).hexdigest() + + +def test_missing_source_does_not_adopt_same_named_destination(sources): + folders, resources = sources + write_wave(folders["destination"] / "voice.wav") + with pytest.raises(ValueError): + resources.prepare_media({"audio_guide": "/api/v1/file/voice.wav?workspace=source"}) + + +def test_non_audio_source_is_rejected(sources): + folders, resources = sources + (folders["source"] / "bad.wav").write_text("not audio") + with pytest.raises(ValueError): + resources.prepare_media({"audio_guide": "/api/v1/file/bad.wav?workspace=source"}) + + +def test_symlink_escape_is_rejected_before_probe(sources, tmp_path): + folders, resources = sources + write_wave(tmp_path / "outside.wav") + (folders["source"] / "voice.wav").symlink_to(tmp_path / "outside.wav") + with pytest.raises(ValueError): + resources.prepare_media({"audio_guide": "/api/v1/file/voice.wav?workspace=source"}) + + +def test_json_flag_cannot_authorize_prepared_voice_paths(sources, tmp_path): + folders, _ = sources + outside = tmp_path / "outside.wav" + write_wave(outside) + with pytest.raises(ValueError): + prepare_generation_inputs({"generation_mode": "audio", "audio_guide": str(outside), + "prepared_studio_speech": True}, + {"returns_audio": True, "wangp_1272": True}, "destination", + workspace_dir=folders["destination"], uploads_dir=folders["uploads"]) + + +@pytest.mark.parametrize("field", ["audio_guide", "audio_guide2", "audio_guide3", "audio_guide4", "audio_guide5", "audio_guide6"]) +def test_every_native_voice_slot_is_resolved_without_the_trusted_adapter(sources, field): + folders, _ = sources + write_wave(folders["uploads"] / "voice.wav") + working = {"generation_mode": "audio", field: "/api/v1/uploads/voice.wav"} + prepare_generation_inputs(working, {"audio_only": True, "wangp_1272": True}, "destination", + workspace_dir=folders["destination"], uploads_dir=folders["uploads"]) + assert working[field] == str(folders["uploads"] / "voice.wav") + + +@pytest.mark.parametrize("reference", ["asset_voice", "/api/v1/assets/asset_voice"]) +def test_audio_asset_identity_preserves_source_and_rejects_ambiguous_locations(sources, reference): + folders, resources = sources + source = folders["source"] / "voice.wav" + write_wave(source) + write_asset_manifest(source, build_asset_manifest(source, asset_id="asset_voice", tool="studio")) + working, identities = resources.prepare_media({"audio_guide": reference}) + assert working["audio_guide"] == str(source) + assert identities[0]["workspace"] == "source" + assert identities[0]["url"] == reference + destination = folders["destination"] / "voice.wav" + write_wave(destination) + write_asset_manifest(destination, build_asset_manifest(destination, asset_id="asset_voice", tool="studio")) + with pytest.raises(ValueError, match="multiple locations"): + resources.prepare_media({"audio_guide": reference}) diff --git a/tests/test_studio_speech_runtime_review.py b/tests/test_studio_speech_runtime_review.py new file mode 100644 index 000000000..c92b1f21d --- /dev/null +++ b/tests/test_studio_speech_runtime_review.py @@ -0,0 +1,195 @@ +"""Provider-free checks for the speech adapter's runtime wiring. + +These tests deliberately stop at the native request boundary. They exercise +the real runtime factory and the real speech preparation adapter, while the +``generate`` callable is a small stand-in for the existing native facade. No +model, worker or provider is loaded. +""" + +from __future__ import annotations + +import asyncio +from copy import deepcopy +import json +from pathlib import Path +import threading + +from services.image_generation_runtime import create_image_generation_commands +from services.native_generation_operation import NativeGenerationOperation +from services.task_manager import TaskRegistry +from services.wangp_submission import JsonRequest, prepare_generation_inputs + + +FIXTURE = Path(__file__).parent / "fixtures" / "studio_speech_native_request.json" + + +def _run(awaitable): + return asyncio.run(awaitable) + + +def _speech_command(intent_id="runtime-speech-intent"): + params = json.loads(FIXTURE.read_text(encoding="utf-8")) + workspace = params.pop("workspace") + return { + "version": 2, + "operation": "generation.speech", + "intent_id": intent_id, + "input": {"workspace": workspace, "params": params}, + } + + +def _task_fields(job): + return { + "id": f"task-{job['id']}", + "root_id": f"task-{job['id']}", + "kind": "audio", + "workflow": "generation", + "title": "Audio generation", + "status": "queued", + "phase": "queued", + "message": "Queued", + "workspace": job["workspace"], + "backend_job_id": job["id"], + "current": 0, + "total": 1, + "resource_requirements": ["local_gpu:0"], + "recoverable": True, + } + + +class _Queue: + def __init__(self): + self.jobs = [] + + def upsert(self, job): + self.jobs.append(deepcopy(job)) + + +class _ExecutionMode: + class ExecutionModeError(RuntimeError): + pass + + def validate_generation(self, _workspace): + return None + + +class _WGP: + primary_settings = {} + + def __init__(self, definition): + self.definition = definition + + def get_model_def(self, model_type): + return deepcopy(self.definition) if model_type == "kugelaudio_0_open" else None + + def get_lora_search_dirs(self, _model_type): + return [] + + +def _runtime(tmp_path, observed): + definition = { + "audio_only": True, + "image_outputs": False, + "inference_steps": False, + "guidance_max_phases": 1, + "duration_slider": {"min": 1, "max": 600, "default": 20}, + "audio_prompt_type_sources": {"selection": ["", "A", "AB"], "default": ""}, + "audio_mode_from_voice_count": True, + "max_voice_count": 6, + } + queue = _Queue() + registry = TaskRegistry(str(tmp_path / "runtime-speech"), interrupt_stale=False) + jobs = {} + dispatched = [] + started = threading.Event() + + def make_job(params, workspace, *, job_id=None, created_at=None, + reserve_generation=False, publish_task=False, provenance=None): + del reserve_generation, publish_task + job = { + "id": job_id or "runtime-speech-job", + "status": "queued", + "created_at": created_at or 1000.0, + "params": deepcopy(params), + "workspace": workspace, + "provenance": deepcopy(provenance or {}), + } + return job + + async def native_generate(request): + body = await request.json() + observed["prepared_speech"] = getattr(request, "prepared_studio_speech", False) + observed["trusted_tool"] = getattr(request, "trusted_tool", None) + observed["body_before_native_boundary"] = deepcopy(body) + # This is the same in-process capability handoff performed by the + # real /api/v1/generate route after its ordinary validation. + workspace = body.pop("workspace") + provenance = body.pop("provenance") + prepare_generation_inputs( + body, + definition, + workspace, + uploads_dir=str(tmp_path / "uploads"), + workspace_dir=str(tmp_path / workspace), + prepared_speech=getattr(request, "prepared_studio_speech", False) is True, + ) + observed["body_after_native_preparation"] = deepcopy(body) + return request.admit_generation_command(body, workspace, provenance) + + runtime = { + "_durable_generation_queue": queue, + "_run_generation_with_preparation": lambda _job_id: started.set(), + "_jobs": jobs, + "register_generation_job": lambda _lock, job: dispatched.append(("registered", deepcopy(job))), + "_gen_lock": object(), + "_cancel_h3_idle_release": lambda: None, + "_active_gen_states": {}, + "_task_registry": lambda _workspace: registry, + "generate": native_generate, + "_new_generation_job": make_job, + "_generation_task_fields": _task_fields, + "execution_mode": _ExecutionMode(), + "wgp": _WGP(definition), + "_check_model_downloaded": lambda _model_type: True, + "_workspace_dir": lambda workspace: str(tmp_path / workspace), + "_list_workspaces": lambda: [{"name": "runtime-speech"}], + "_lora_is_compatible_with_model": lambda _definition, _path: True, + } + service = create_image_generation_commands(runtime) + return service, registry, queue, dispatched, started + + +def test_runtime_factory_speech_adapter_reaches_native_boundary_once(tmp_path): + observed = {} + service, registry, queue, dispatched, started = _runtime(tmp_path, observed) + + adapter = service.operations["generation.speech"] + assert isinstance(adapter, NativeGenerationOperation) + assert adapter.catalog["name"] == "generation.speech" + + command = _speech_command() + result = _run(service.submit(command, trusted_tool="external_agent")) + + assert result["replayed"] is False + assert result["receipt"]["operation"] == "generation.speech" + assert observed["prepared_speech"] is True + assert observed["trusted_tool"] == "external_agent" + assert observed["body_before_native_boundary"]["generation_mode"] == "audio" + assert observed["body_after_native_preparation"]["generation_mode"] == "audio" + assert len(dispatched) == 1 + assert started.wait(1) + assert len(queue.jobs) == 1 + + entry = registry.command_admission(command["intent_id"]) + assert entry is not None + assert entry["operation"] == "generation.speech" + assert entry["receipt"] == result["receipt"] + assert entry["effective"]["runtime"]["params"]["prompt"] == command["input"]["params"]["prompt"] + + +def test_json_payload_cannot_forge_prepared_speech_capability(): + request = JsonRequest({"prepared_studio_speech": True}) + + # The marker is an in-process capability set by ImageGenerationCommands; + # arbitrary JSON keys must not become trusted request attributes. + assert getattr(request, "prepared_studio_speech", False) is False diff --git a/tests/test_studio_speech_spec.py b/tests/test_studio_speech_spec.py new file mode 100644 index 000000000..969a343b0 --- /dev/null +++ b/tests/test_studio_speech_spec.py @@ -0,0 +1,246 @@ +"""Provider-free tests for the closed Studio speech command envelope.""" + +from copy import deepcopy +import hashlib +import json +from pathlib import Path + +import pytest + +from services.studio_speech_spec import ( + STUDIO_SPEECH_DEFAULTS, + StudioSpeechSpecError, + freeze_studio_speech_spec, + studio_speech_schema, +) + + +FIXTURE = Path(__file__).parent / "fixtures" / "studio_speech_native_request.json" + + +def speech_command(intent="speech-intent"): + native = json.loads(FIXTURE.read_text()) + workspace = native.pop("workspace") + return { + "version": 2, + "operation": "generation.speech", + "intent_id": intent, + "input": {"workspace": workspace, "params": native}, + } + + +def test_captured_studio_body_round_trips_without_trimming_or_dropping_fields(): + command = speech_command() + before = deepcopy(command) + frozen = freeze_studio_speech_spec(command) + + assert command == before + assert frozen["original"] == before + assert frozen["original"] is not command + assert frozen["original"]["input"]["params"]["prompt"] == "The system is watching.\nEvery warning matters." + assert frozen["effective"]["input"]["params"]["_tts_original_prompt"] == before["input"]["params"]["_tts_original_prompt"] + assert frozen["effective"]["input"]["params"]["duration_seconds"] == 20.0 + assert set(before["input"]["params"]) <= set(frozen["effective"]["input"]["params"]) + + +def test_effective_defaults_are_native_speech_selectors_and_original_omissions_survive(): + command = speech_command() + params = command["input"]["params"] + for key in ( + "generation_mode", + "_audio_sub_mode", + "video_length", + "image_mode", + "multi_prompts_gen_type", + "negative_prompt", + "repeat_generation", + "activated_loras", + "loras_multipliers", + ): + params.pop(key, None) + params.pop("_tts_original_prompt") + frozen = freeze_studio_speech_spec(command) + effective = frozen["effective"]["input"]["params"] + + assert "generation_mode" not in command["input"]["params"] + assert "_tts_original_prompt" not in command["input"]["params"] + for key, value in STUDIO_SPEECH_DEFAULTS.items(): + assert effective[key] == value + assert effective["_tts_original_prompt"] == params["prompt"] + + +@pytest.mark.parametrize("value", ["cinematic", 1, False, [], {}]) +def test_prompt_enhancer_is_rejected_when_active_or_wrongly_typed(value): + command = speech_command() + command["input"]["params"]["prompt_enhancer"] = value + + with pytest.raises(StudioSpeechSpecError): + freeze_studio_speech_spec(command) + + +@pytest.mark.parametrize("value", ["", None]) +def test_prompt_enhancer_inactive_sentinels_are_preserved(value): + command = speech_command() + command["input"]["params"]["prompt_enhancer"] = value + + frozen = freeze_studio_speech_spec(command) + + assert frozen["effective"]["input"]["params"]["prompt_enhancer"] == value + + +def test_fingerprint_excludes_transport_intent_but_covers_workspace_and_native_content(): + first = freeze_studio_speech_spec(speech_command("one")) + second = freeze_studio_speech_spec(speech_command("two")) + assert first["fingerprint"] == second["fingerprint"] + changed = speech_command("three") + changed["input"]["params"]["prompt"] += " changed" + assert freeze_studio_speech_spec(changed)["fingerprint"] != first["fingerprint"] + other_workspace = speech_command("four") + other_workspace["input"]["workspace"] = "another-workspace" + assert freeze_studio_speech_spec(other_workspace)["fingerprint"] != first["fingerprint"] + + +def test_fingerprint_is_stable_and_uses_canonical_json(): + frozen = freeze_studio_speech_spec(speech_command()) + content = { + "version": 2, + "operation": "generation.speech", + "input": frozen["effective"]["input"], + } + expected = hashlib.sha256( + json.dumps(content, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + ).hexdigest() + assert frozen["fingerprint"] == expected + assert len(frozen["fingerprint"]) == 64 + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("generation_mode", "image"), + ("_audio_sub_mode", "music"), + ("image_mode", 1), + ("video_length", 1), + ("multi_prompts_gen_type", 1), + ("minimax_h3_turbo_mode", True), + ("audio_prompt_type", "A/host-path"), + ], +) +def test_speech_mode_rejects_video_music_and_active_h3_values(field, value): + command = speech_command() + command["input"]["params"][field] = value + with pytest.raises(StudioSpeechSpecError): + freeze_studio_speech_spec(command) + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("num_inference_steps", True), + ("num_inference_steps", 1.0), + ("_tts_voice_count", "1"), + ("_tts_voice_count", True), + ("duration_seconds", "20"), + ("duration_seconds", float("nan")), + ("temperature", True), + ("top_k", 2.5), + ("tts_dynaudnorm", 2), + ], +) +def test_native_scalars_are_strict_and_finite(field, value): + command = speech_command() + command["input"]["params"][field] = value + with pytest.raises(StudioSpeechSpecError): + freeze_studio_speech_spec(command) + + +@pytest.mark.parametrize( + "value", + [ + "/etc/passwd", + "../voice.wav", + "C:\\voice.wav", + "https://example.test/voice.wav", + "/api/v1/file/voice.wav", + "/api/v1/file/../voice.wav?workspace=speech-test", + "/api/v1/uploads/voice.wav?workspace=other", + "asset_abc?workspace=other", + ], +) +def test_audio_references_are_canonical_and_workspace_explicit(value): + command = speech_command() + command["input"]["params"].update( + {"audio_prompt_type": "A", "_tts_voice_count": 1, "audio_guide": value} + ) + with pytest.raises(StudioSpeechSpecError): + freeze_studio_speech_spec(command) + + +@pytest.mark.parametrize( + "value", + [ + "", + None, + "asset_abc123", + "/api/v1/uploads/voice.wav", + "/api/v1/file/voice.wav?workspace=speech-test", + "/api/v1/assets/asset_abc123", + ], +) +def test_audio_reference_sentinels_and_canonical_urls_are_retained(value): + command = speech_command() + command["input"]["params"].update( + {"audio_prompt_type": "", "_tts_voice_count": 0, "audio_guide": value} + ) + frozen = freeze_studio_speech_spec(command) + assert frozen["original"]["input"]["params"]["audio_guide"] == value + assert frozen["effective"]["input"]["params"]["audio_guide"] == value + + +def test_inactive_shared_state_is_retained_only_as_empty_sentinels(): + command = speech_command() + params = command["input"]["params"] + params.update( + { + "image_start": ["", ""], + "image_refs": [], + "h3_ref_videos": [], + "spatial_upsampling": "", + "wangp_processor_settings": {}, + "MMAudio_setting": 0, + "MMAudio_prompt": None, + } + ) + frozen = freeze_studio_speech_spec(command) + assert frozen["effective"]["input"]["params"]["image_start"] == ["", ""] + assert frozen["effective"]["input"]["params"]["MMAudio_setting"] == 0 + params["image_start"] = ["/api/v1/uploads/image.png"] + with pytest.raises(StudioSpeechSpecError): + freeze_studio_speech_spec(command) + + +def test_custom_settings_are_closed_and_authority_fields_are_rejected(): + command = speech_command() + command["input"]["params"]["custom_settings"] = {"auto_split_every_s": 5} + frozen = freeze_studio_speech_spec(command) + assert frozen["effective"]["input"]["params"]["custom_settings"] == {"auto_split_every_s": 5.0} + for key, value in (("unknown", 1), ("provenance", {}), ("actor", "user")): + invalid = speech_command() + if key in {"provenance", "actor"}: + invalid["input"]["params"][key] = value + else: + invalid["input"]["params"]["custom_settings"] = {key: value} + with pytest.raises(StudioSpeechSpecError): + freeze_studio_speech_spec(invalid) + + +def test_schema_exposes_closed_speech_boundary_and_supported_models(): + schema = studio_speech_schema() + assert schema["version"] == 2 + assert schema["operation"] == "generation.speech" + params_schema = schema["input"]["$defs"]["StudioSpeechParams"] + assert params_schema["additionalProperties"] is False + assert "_tts_original_prompt" in params_schema["properties"] + assert "kugelaudio_0_open" in schema["speech_model_types"] + assert "minimax_music3" not in schema["speech_model_types"] + assert "provenance" in schema["excluded"] diff --git a/tests/test_tools_command_runtime.py b/tests/test_tools_command_runtime.py new file mode 100644 index 000000000..f09bbeb7a --- /dev/null +++ b/tests/test_tools_command_runtime.py @@ -0,0 +1,174 @@ +"""Native Tools route admission and recovery; no model or processor execution.""" +import ast +from copy import deepcopy +from pathlib import Path +from types import SimpleNamespace +import threading +import time +import uuid + +from fastapi import FastAPI, HTTPException +from fastapi.testclient import TestClient +import pytest + +from routers.image_generation_commands import create_image_generation_commands_router, image_command_handlers +from routers.tools_upscale_commands import tools_upscale_command_catalog +from services.native_generation_operation import NativeGenerationOperation +from services.tools_upscale import TOOL_UPSCALE_METHODS +from services.tools_upscale_spec import freeze_tools_upscale_spec +from tests.test_image_generation_commands import FakeNative, _command as image_command, _run + + +def command(intent="upscale-intent"): + return {"version": 2, "operation": "tools.upscale", "intent_id": intent, + "input": {"workspace": "tool-destination", "params": { + "source": "/api/v1/file/source.png?workspace=tool-source", + "source_kind": "image", "method": "lanczos2", "seed": 42}}} + + +def native_endpoint(tmp_path, legacy_calls): + source = Path(__file__).resolve().parents[1] / "app" / "_launch_runtime.py" + tree = ast.parse(source.read_text(encoding="utf-8")) + definition = next(node for node in tree.body if isinstance(node, ast.AsyncFunctionDef) and node.name == "tools_upscale") + definition.decorator_list = [] + namespace = { + "Request": object, "HTTPException": HTTPException, "uuid": uuid, "time": time, + "threading": threading, "_TOOL_UPSCALE_METHODS": TOOL_UPSCALE_METHODS, + "_resolve_tool_source": lambda body, **_kwargs: ( + str(tmp_path / "source.png"), "source.png", "tool-source", body["source_kind"], + "asset_source", body["workspace"], str(tmp_path / "destination")), + "_register_manual_generation_job": lambda job: legacy_calls.append(job), + "execution_mode": SimpleNamespace(policy=lambda: SimpleNamespace(simulated=False)), + } + exec(compile(ast.Module(body=[definition], type_ignores=[]), str(source), "exec"), namespace) + return namespace["tools_upscale"] + + +def configured_service(native, tmp_path): + service = native.service() + legacy_calls, worker_calls = [], [] + + def freeze(body): + frozen = freeze_tools_upscale_spec(body) + original_input = frozen["effective"]["input"] + return frozen, {**deepcopy(original_input["params"]), "workspace": original_input["workspace"]} + + service.runtime_defaults = lambda: {"prompt": "unrelated image form", "model_type": "must-not-leak"} + service.operations["tools.upscale"] = NativeGenerationOperation( + freeze=freeze, prepare=lambda params: (params, []), catalog=tools_upscale_command_catalog(), + prepare_request=native_endpoint(tmp_path, legacy_calls), + worker=lambda job_id: worker_calls.append(job_id) or True, use_generation_defaults=False, + ) + return service, legacy_calls, worker_calls + + +def test_tools_http_and_mcp_share_the_native_route_and_exact_receipt(tmp_path): + native = FakeNative(tmp_path) + service, legacy, workers = configured_service(native, tmp_path) + app = FastAPI() + app.include_router(create_image_generation_commands_router(service)) + with TestClient(app) as client: + response = client.post("/api/v1/generation/commands", json=command(), + headers={"X-Hocus-UI-Surface": "wizard"}) + assert response.status_code == 200 + first = response.json() + second = _run(image_command_handlers(service)["tools.upscale"]( + {key: value for key, value in command().items() if key != "operation"})) + assert first["receipt"] == second["receipt"] + assert second["replayed"] is True + assert len(native.dispatch_calls) == 1 + job = native.dispatch_calls[0] + assert job["params"]["model_type"] == "post_processing" + assert "prompt" not in job["params"] + assert "_non_durable_tool" not in job["params"] + assert job["params"]["source_workspace"] == "tool-source" + assert job["provenance"]["capability"] == "tools.upscale" + assert job["provenance"]["actor"] == "wizard" + assert service.native_worker(job)(job["id"]) is True + assert workers == [job["id"]] + assert legacy == [] + assert native.prepare_calls == 0 + another = _run(service.submit(command("another-deliberate-upscale"))) + assert another["receipt"]["taskIds"] != first["receipt"]["taskIds"] + + +def test_tool_recovery_retains_native_snapshot_and_requires_exact_admission(tmp_path): + native = FakeNative(tmp_path) + service, _, _ = configured_service(native, tmp_path) + accepted = _run(service.submit(command())) + restarted = FakeNative(tmp_path, interrupt_stale=True) + recovery, legacy, workers = configured_service(restarted, tmp_path) + recovery.restore_recovery(["tool-destination"]) + assert len(restarted.persist_calls) == 1 + record = restarted.persist_calls[0] + assert recovery.filter_recovery([record]) == [record] + assert record["params"] == native.dispatch_calls[0]["params"] + assert recovery.native_worker(record)(record["id"]) is True + assert workers == [record["id"]] + forged = deepcopy(record) + forged["provenance"]["command"]["command_id"] = "not-admitted" + with pytest.raises(HTTPException) as caught: + recovery.native_worker(forged) + assert caught.value.status_code == 503 + assert workers == [record["id"]] + drifted = deepcopy(record) + drifted["id"] = "different-native-job" + with pytest.raises(HTTPException) as mismatch: + recovery.native_worker(drifted) + assert mismatch.value.status_code == 503 + assert mismatch.value.detail["code"] == "recovery_mismatch" + assert workers == [record["id"]] + assert legacy == [] + assert _run(recovery.submit(command()))["receipt"] == accepted["receipt"] + assert restarted.dispatch_calls == [] + + +def test_tool_keeps_collection_attribution_separate_from_physical_output_workspace(tmp_path): + native = FakeNative(tmp_path) + service, _, _ = configured_service(native, tmp_path) + request = command() + request["input"]["workspace_collection_id"] = "collection-curated" + accepted = _run(service.submit(request)) + job = native.dispatch_calls[0] + assert job["workspace"] == "tool-destination" + assert accepted["receipt"]["result"]["workspace"] == "tool-destination" + assert job["provenance"]["workspace_id"] == "collection-curated" + + +def test_image_admission_cannot_be_relabelled_as_a_tool_worker(tmp_path): + native = FakeNative(tmp_path) + service, _, workers = configured_service(native, tmp_path) + _run(service.submit(image_command())) + record = deepcopy(native.dispatch_calls[0]) + record["provenance"]["capability"] = "tools.upscale" + with pytest.raises(HTTPException): + service.native_worker(record) + assert workers == [] + + +@pytest.mark.parametrize("capability", ["upscale", "tools.upscale"]) +def test_native_task_projection_identifies_upscale_for_activity(tmp_path, capability): + source = Path(__file__).resolve().parents[1] / "app" / "_launch_runtime.py" + tree = ast.parse(source.read_text(encoding="utf-8")) + definition = next(node for node in tree.body if isinstance(node, ast.FunctionDef) + and node.name == "_generation_task_fields") + namespace = { + "time": time, + "_public_generation_details": lambda params: params, + "_task_status": lambda status: status, + "_task_timestamp": lambda job, key: job.get(key), + "_canonical_legacy_progress": lambda *_args: 0, + "_is_durable_generation_job": lambda _job: True, + "_local_gpu_lane": SimpleNamespace(key="local_gpu:0"), + } + exec(compile(ast.Module(body=[definition], type_ignores=[]), str(source), "exec"), namespace) + task = namespace["_generation_task_fields"]({ + "id": "upscale-job", "status": "queued", "workspace": "tool-destination", + "params": {"generation_mode": "image", "model_type": "post_processing"}, + "provenance": {"capability": capability, "command": {"command_id": "tool-intent"}}, + }) + assert task["title"] == "Tools · Upscale" + assert task["metadata"]["capability"] == capability + assert task["metadata"]["command_id"] == "tool-intent" + assert task["workspace"] == "tool-destination" + assert task["backend_job_id"] == "upscale-job" diff --git a/tests/test_tools_upscale_preparation.py b/tests/test_tools_upscale_preparation.py new file mode 100644 index 000000000..6af2d6e0a --- /dev/null +++ b/tests/test_tools_upscale_preparation.py @@ -0,0 +1,404 @@ +"""Provider-free source and processor preparation checks for Tools upscale.""" + +from copy import deepcopy +import hashlib + +import pytest +from fastapi import HTTPException +from PIL import Image + +from services.asset_manifest import build_asset_manifest, write_asset_manifest +from services.tools_upscale_commands import ( + _canonical_request_source, + _request_ready_params, + _resolve_source, +) +from services.tools_upscale_preparation import prepare_tools_upscale + + +class FakeResources: + def __init__(self, roots, references): + self.roots = roots + self.references = references + + def workspace_dir(self, name): + return str(self.roots[name]) + + def uploads_dir(self): + return str(self.roots["__uploads__"]) + + def _media(self, value): + path, workspace = self.references[value] + return str(path), workspace + + +@pytest.fixture +def prepared_fixture(tmp_path): + roots = {name: tmp_path / name for name in ("source", "destination", "__uploads__")} + for root in roots.values(): + root.mkdir() + source = roots["source"] / "poster.png" + reference = roots["source"] / "face.png" + Image.new("RGB", (19, 13), "navy").save(source) + Image.new("RGB", (7, 11), "orange").save(reference) + source_url = "/api/v1/file/poster.png?workspace=source" + reference_url = "/api/v1/file/face.png?workspace=source" + resources = FakeResources( + roots, + {source_url: (source, "source"), reference_url: (reference, "source")}, + ) + + def resolver(_params, **_kwargs): + return str(source), source.name, "source", "image", "asset-poster", "destination", str(roots["destination"]) + + def capabilities(): + return [{"value": "lanczos2", "kind": "spatial", "media": ("image",), "enabled": True}] + + def validate(spatial, temporal, image): + assert (spatial, temporal, image) == ("lanczos2", "", True) + return "" + + def settings(method, values): + assert method == "lanczos2" + return dict(values) + + return { + "roots": roots, + "source": source, + "reference": reference, + "source_url": source_url, + "reference_url": reference_url, + "resources": resources, + "resolver": resolver, + "capabilities": capabilities, + "validate": validate, + "settings": settings, + } + + +def _params(fixture, **extra): + params = { + "workspace": "destination", + "source": fixture["source_url"], + "source_kind": "image", + "method": "lanczos2", + "seed": -1, + "wangp_processor_settings": {}, + } + params.update(extra) + return params + + +def _prepare(fixture, params, **extra): + kwargs = { + "resources": fixture["resources"], + "resolve_source": fixture["resolver"], + "processor_capabilities": fixture["capabilities"], + "validate_processors": fixture["validate"], + "processor_settings": fixture["settings"], + } + kwargs.update(extra) + return prepare_tools_upscale(params, **kwargs) + + +def test_image_source_is_confined_inspected_and_snapshotted(prepared_fixture): + params = _params(prepared_fixture) + before = deepcopy(params) + native, resources = _prepare(prepared_fixture, params) + + assert params == before + assert native["source_path"] == str(prepared_fixture["source"]) + assert native["source_workspace"] == "source" + assert native["source_asset_id"] == "asset-poster" + assert native["source_kind"] == "image" + assert native["generation_mode"] == "image" + assert resources[0]["role"] == "source" + assert resources[0]["workspace"] == "source" + assert resources[0]["media"]["width"] == 19 + assert resources[0]["sha256"] + assert resources[0]["size_bytes"] == prepared_fixture["source"].stat().st_size + + +def test_source_workspace_is_forwarded_and_must_match_resolved_source(prepared_fixture): + seen = {} + + def resolver(request, **_kwargs): + seen.update(request) + return ( + str(prepared_fixture["source"]), prepared_fixture["source"].name, + "source", "image", "asset-poster", "destination", + str(prepared_fixture["roots"]["destination"]), + ) + + native, _ = _prepare( + prepared_fixture, + _params(prepared_fixture, source_workspace="source"), + resolve_source=resolver, + ) + + assert seen["source_workspace"] == "source" + assert native["source_workspace"] == "source" + + with pytest.raises(HTTPException, match="source_workspace"): + _prepare( + prepared_fixture, + _params(prepared_fixture, source_workspace="destination"), + resolve_source=resolver, + ) + + +def test_asset_scope_preflight_rejects_ambiguity_and_keeps_explicit_identity(prepared_fixture): + destination = prepared_fixture["roots"]["destination"] / "poster.png" + Image.new("RGB", (19, 13), "white").save(destination) + for path, workspace in ((prepared_fixture["source"], "source"), (destination, "destination")): + write_asset_manifest( + path, + build_asset_manifest(path, asset_id="asset-shared", workspace_id=workspace, tool="fixture"), + ) + calls = [] + + def native_resolver(body, **_kwargs): + calls.append(deepcopy(body)) + path = prepared_fixture["source"] if body["source_workspace"] == "source" else destination + return str(path), path.name, body["source_workspace"], "image", "asset-shared", "destination", str( + prepared_fixture["roots"]["destination"] + ) + + resolve = _resolve_source({ + "_resolve_tool_source": native_resolver, + "_tool_asset_roots": lambda: [ + {"workspace_id": name, "path": str(path)} + for name, path in prepared_fixture["roots"].items() + ], + }) + request = { + "source": "/api/v1/assets/asset-shared", + "source_kind": "image", + "workspace": "destination", + } + with pytest.raises(HTTPException) as ambiguous: + resolve(request) + assert ambiguous.value.status_code == 409 + assert ambiguous.value.detail["code"] == "ambiguous_source" + assert calls == [] + + with pytest.raises(HTTPException) as missing: + resolve({**request, "source_workspace": "missing"}) + assert missing.value.status_code == 409 + assert missing.value.detail["code"] == "source_workspace_mismatch" + assert calls == [] + + native, resources = _prepare( + prepared_fixture, + _params( + prepared_fixture, + source="/api/v1/assets/asset-shared", + source_workspace="source", + ), + resolve_source=resolve, + ) + assert calls[-1] == { + "source_kind": "image", + "workspace": "destination", + "asset_id": "asset-shared", + "source_workspace": "source", + } + assert native["source_workspace"] == "source" + assert native["source_asset_id"] == "asset-shared" + source_hash = hashlib.sha256(prepared_fixture["source"].read_bytes()).hexdigest() + destination_hash = hashlib.sha256(destination.read_bytes()).hexdigest() + assert source_hash != destination_hash + assert resources[0]["sha256"] == source_hash + + +def test_processor_reference_is_resolved_before_validator_and_keeps_order(prepared_fixture): + seen = {} + + def settings(method, values): + seen["values"] = deepcopy(values) + return {"spatial_upsampler_reference_images": values["spatial_upsampler_reference_images"]} + + # A small valid MP4 is unnecessary here: the injected probe is the + # provider-free media boundary, and the source bytes are still hashed. + clip = prepared_fixture["roots"]["source"] / "poster.mp4" + clip.write_bytes(b"fixture video") + native, resources = _prepare( + prepared_fixture, + _params( + prepared_fixture, + method="h3facerefine", + source_kind="video", + source="/api/v1/file/poster.mp4?workspace=source", + wangp_processor_settings={ + "spatial_upsampler_reference_images": [prepared_fixture["reference_url"]], + }, + ), + processor_capabilities=lambda: [{ + "value": "h3facerefine", "kind": "spatial", "media": ("video",), "enabled": True, + }], + validate_processors=lambda spatial, temporal, image: "", + processor_settings=settings, + processor_parameters=lambda _method: [ + {"name": "spatial_upsampler_reference_images", "type": "array"}, + ], + resolve_source=lambda _params, **_kwargs: ( + str(clip), clip.name, "source", "video", "asset-clip", "destination", + str(prepared_fixture["roots"]["destination"]), + ), + probe_video=lambda _path: {"duration": 1.5, "width": 96, "height": 54, "fps": 24}, + ) + + assert native["source_kind"] == "video" + assert seen["values"]["spatial_upsampler_reference_images"] == [ + str(prepared_fixture["reference"]) + ] + assert native["wangp_processor_settings"]["spatial_upsampler_reference_images"] == [ + str(prepared_fixture["reference"]) + ] + assert [item["role"] for item in resources] == ["source", "processor_reference"] + assert resources[1]["index"] == 0 + assert resources[1]["workspace"] == "source" + + +def test_video_source_requires_a_positive_probe(prepared_fixture): + clip = prepared_fixture["roots"]["source"] / "poster.mp4" + clip.write_bytes(b"fixture video") + + def resolver(_params, **_kwargs): + return str(clip), clip.name, "source", "video", "asset-clip", "destination", None + + with pytest.raises(HTTPException, match="duration"): + _prepare( + prepared_fixture, + _params( + prepared_fixture, + source="/api/v1/file/poster.mp4?workspace=source", + source_kind="video", + ), + resolve_source=resolver, + processor_capabilities=lambda: [], + validate_processors=lambda *_args: "", + processor_settings=lambda _method, values: dict(values), + probe_video=lambda _path: {"duration": 0, "width": 96, "height": 54}, + ) + + +def test_invalid_image_is_rejected_before_native_snapshot(prepared_fixture): + invalid = prepared_fixture["roots"]["source"] / "invalid.png" + invalid.write_bytes(b"not an image") + + with pytest.raises(HTTPException, match="could not be decoded"): + _prepare( + prepared_fixture, + _params(prepared_fixture, source="/api/v1/file/invalid.png?workspace=source"), + resolve_source=lambda _params, **_kwargs: ( + str(invalid), invalid.name, "source", "image", "asset-invalid", "destination", None, + ), + ) + + +def test_disabled_or_wrong_media_processor_is_rejected(prepared_fixture): + with pytest.raises(HTTPException, match="disabled"): + _prepare( + prepared_fixture, + _params(prepared_fixture), + processor_capabilities=lambda: [{ + "value": "lanczos2", "kind": "spatial", "media": ("image",), "enabled": False, + }], + ) + + +def test_processor_unknown_setting_is_rejected_without_mutating_input(prepared_fixture): + params = _params(prepared_fixture, wangp_processor_settings={"spatial_upsampler_prompt": "literal"}) + before = deepcopy(params) + with pytest.raises(HTTPException, match="not declared"): + _prepare(prepared_fixture, params, processor_settings=lambda _method, _values: {}) + assert params == before + + +def test_factory_source_adapter_accepts_prepared_asset_id_without_losing_it(): + seen = [] + + def resolver(body, **kwargs): + seen.append((body, kwargs)) + return "resolved" + + resolve = _resolve_source({"_resolve_tool_source": resolver}) + result = resolve({ + "source_kind": "image", + "workspace": "destination", + "asset_id": "asset-poster", + }, expected_kinds=("image",)) + + assert result == "resolved" + assert seen == [( + { + "source_kind": "image", + "workspace": "destination", + "asset_id": "asset-poster", + }, + {"expected_kinds": ("image",)}, + )] + + +def test_scoped_file_source_does_not_depend_on_global_asset_discovery(): + def unavailable_catalog(): + raise OSError("An unrelated catalog root is unavailable") + + request = { + "source": "/api/v1/file/poster.png?workspace=source", + "source_kind": "image", "workspace": "destination", + } + resolve = _resolve_source({ + "_resolve_tool_source": lambda body, **_kwargs: body, + "_tool_asset_roots": unavailable_catalog, + }) + assert resolve(request) == request + + +def test_native_projection_keeps_managed_asset_id_with_canonical_source(tmp_path): + source_root = tmp_path / "source" + output_root = tmp_path / "destination" + uploads_root = tmp_path / "uploads" + source_root.mkdir() + output_root.mkdir() + uploads_root.mkdir() + source_path = source_root / "poster.png" + source_path.write_bytes(b"already inspected") + params = { + "source": "asset-poster", + "source_kind": "image", + "workspace": "destination", + "source_path": str(source_path), + "source_workspace": "source", + "source_asset_id": "asset-poster", + } + + ready = _request_ready_params(params) + canonical = _canonical_request_source( + params, + { + "_workspace_dir": lambda workspace: str( + {"source": source_root, "destination": output_root}[workspace] + ), + "_uploads_dir": lambda: str(uploads_root), + }, + ) + ready.pop("source_path", None) + ready["source"] = canonical + + assert canonical == "/api/v1/file/poster.png?workspace=source" + assert ready["asset_id"] == "asset-poster" + + +@pytest.mark.parametrize("settings", [None, {}, {"spatial_upsampler_prompt": None}]) +def test_empty_settings_cross_real_scalar_processor_validator(prepared_fixture, settings): + from shared.wangp1272.processors import validated_settings + + native, _ = _prepare( + prepared_fixture, + _params(prepared_fixture, wangp_processor_settings=settings), + processor_settings=validated_settings, + processor_parameters=lambda _method: [], + ) + assert native["wangp_processor_settings"] == {} diff --git a/tests/test_tools_upscale_spec.py b/tests/test_tools_upscale_spec.py new file mode 100644 index 000000000..107893c02 --- /dev/null +++ b/tests/test_tools_upscale_spec.py @@ -0,0 +1,183 @@ +"""Provider-free regressions for the typed Tools upscale envelope.""" + +from copy import deepcopy + +import pytest + +from services.tools_upscale_spec import ( + ToolsUpscaleSpecError, + freeze_tools_upscale_spec, + tools_upscale_schema, +) + + +def command(*, intent="intent-a", source="/api/v1/file/poster.png?workspace=source", + source_kind="image", method="lanczos2", **params): + native = {"source": source, "source_kind": source_kind, "method": method, **params} + return { + "version": 2, + "operation": "tools.upscale", + "intent_id": intent, + "input": {"workspace": "destination", "params": native}, + } + + +def test_freeze_detaches_original_and_adds_only_deterministic_defaults(): + submitted = command() + before = deepcopy(submitted) + frozen = freeze_tools_upscale_spec(submitted) + + assert frozen["original"] == before + assert frozen["original"] is not submitted + assert frozen["effective"]["input"]["params"]["seed"] == -1 + assert frozen["effective"]["input"]["params"]["wangp_processor_settings"] == {} + assert frozen["original"]["input"]["params"] == before["input"]["params"] + submitted["input"]["params"]["source"] = "/api/v1/uploads/other.png" + assert frozen["original"] == before + + +def test_fingerprint_excludes_intent_but_includes_source_kind_and_method(): + first = freeze_tools_upscale_spec(command(intent="one")) + second = freeze_tools_upscale_spec(command(intent="two")) + different_method = freeze_tools_upscale_spec(command(intent="three", method="lanczos1.5")) + different_kind = freeze_tools_upscale_spec( + command(intent="four", source="/api/v1/file/shot.mp4?workspace=source", source_kind="video") + ) + + assert first["fingerprint"] == second["fingerprint"] + assert first["fingerprint"] != different_method["fingerprint"] + assert first["fingerprint"] != different_kind["fingerprint"] + + +def test_source_workspace_is_preserved_and_fingerprinted(): + source = "/api/v1/assets/asset-shared" + source_copy = freeze_tools_upscale_spec(command(source=source, source_workspace="source")) + other_scope = freeze_tools_upscale_spec( + command(intent="other-scope", source=source, source_workspace="default") + ) + + assert source_copy["effective"]["input"]["params"]["source_workspace"] == "source" + assert source_copy["original"]["input"]["params"]["source_workspace"] == "source" + assert source_copy["fingerprint"] != other_scope["fingerprint"] + + +@pytest.mark.parametrize("source,source_workspace", [ + ("/api/v1/file/poster.png?workspace=source", "default"), + ("/api/v1/uploads/poster.png", "source"), +]) +def test_source_workspace_must_match_canonical_url(source, source_workspace): + with pytest.raises(ToolsUpscaleSpecError, match="source_workspace"): + freeze_tools_upscale_spec(command(source=source, source_workspace=source_workspace)) + + +@pytest.mark.parametrize("source_workspace", ["__uploads__", "source", "default"]) +def test_source_workspace_accepts_known_scope_shapes(source_workspace): + source = "/api/v1/uploads/poster.png" if source_workspace == "__uploads__" else "asset-poster" + frozen = freeze_tools_upscale_spec(command(source=source, source_workspace=source_workspace)) + assert frozen["effective"]["input"]["params"]["source_workspace"] == source_workspace + + +@pytest.mark.parametrize("source_workspace", ["", " ", "../source", "/tmp/source", "s" * 161, True, 12]) +def test_source_workspace_is_strict_and_bounded(source_workspace): + with pytest.raises(ToolsUpscaleSpecError, match="source_workspace"): + freeze_tools_upscale_spec(command(source_workspace=source_workspace)) + + +def test_collection_identity_is_preserved_and_fingerprinted(): + first_command = command() + first_command["input"]["workspace_collection_id"] = "collection-one" + second_command = command(intent="different-intent") + second_command["input"]["workspace_collection_id"] = "collection-one" + different_collection = command(intent="another-intent") + different_collection["input"]["workspace_collection_id"] = "collection-two" + + first = freeze_tools_upscale_spec(first_command) + second = freeze_tools_upscale_spec(second_command) + other = freeze_tools_upscale_spec(different_collection) + + assert first["original"]["input"]["workspace_collection_id"] == "collection-one" + assert first["effective"]["input"]["workspace_collection_id"] == "collection-one" + assert "workspace_collection_id" not in first["effective"]["input"]["params"] + assert first["fingerprint"] == second["fingerprint"] + assert first["fingerprint"] != other["fingerprint"] + first_command["input"]["workspace_collection_id"] = "changed-after-freeze" + assert first["effective"]["input"]["workspace_collection_id"] == "collection-one" + + +def test_collection_identity_distinguishes_omission_from_explicit_null(): + omitted = freeze_tools_upscale_spec(command()) + explicit_null_command = command(intent="explicit-null") + explicit_null_command["input"]["workspace_collection_id"] = None + explicit_null = freeze_tools_upscale_spec(explicit_null_command) + + assert "workspace_collection_id" not in omitted["effective"]["input"] + assert explicit_null["effective"]["input"]["workspace_collection_id"] is None + assert omitted["fingerprint"] != explicit_null["fingerprint"] + + +@pytest.mark.parametrize("value", ["", " ", True, 12, [], {}]) +def test_collection_identity_is_strict_and_nonblank(value): + values = command() + values["input"]["workspace_collection_id"] = value + with pytest.raises(ToolsUpscaleSpecError, match="workspace_collection_id"): + freeze_tools_upscale_spec(values) + + +@pytest.mark.parametrize( + "bad", + [ + {"source": "/tmp/poster.png"}, + {"source": "https://remote.invalid/poster.png?workspace=source"}, + {"source": "/api/v1/file/../poster.png?workspace=source"}, + {"source": "/api/v1/file/poster.png?workspace=source&workspace=source"}, + {"source": "/api/v1/file/poster.png"}, + {"source": "/api/v1/uploads/poster.png?workspace=source"}, + {"method": "unknown"}, + {"seed": True}, + {"seed": 1.0}, + {"wangp_processor_settings": {"unknown": 1}}, + {"wangp_processor_settings": {"spatial_upsampler_strength": float("nan")}}, + ], +) +def test_freeze_rejects_noncanonical_or_untyped_values_before_effects(bad): + values = command() + values["input"]["params"].update(bad) + with pytest.raises(ToolsUpscaleSpecError): + freeze_tools_upscale_spec(values) + + +def test_video_only_method_is_rejected_for_image_source(): + with pytest.raises(ToolsUpscaleSpecError, match="video source"): + freeze_tools_upscale_spec(command(method="rife2")) + + +def test_processor_prompt_and_references_are_value_preserving(): + prompt = "Face line 1\n Face line 2 " + source = "/api/v1/file/ref.png?workspace=source" + frozen = freeze_tools_upscale_spec(command( + method="h3facerefine", + source_kind="video", + source="/api/v1/file/shot.mp4?workspace=source", + wangp_processor_settings={ + "spatial_upsampler_prompt": prompt, + "spatial_upsampler_reference_images": [source], + }, + )) + + settings = frozen["effective"]["input"]["params"]["wangp_processor_settings"] + assert settings["spatial_upsampler_prompt"] == prompt + assert settings["spatial_upsampler_reference_images"] == [source] + + +def test_schema_publishes_only_the_closed_tools_surface(): + schema = tools_upscale_schema() + params = schema["input"]["$defs"]["ToolsUpscaleParams"] + assert schema["version"] == 2 + assert schema["operation"] == "tools.upscale" + assert params["additionalProperties"] is False + assert set(schema["supported_input_fields"]) == { + "workspace", "workspace_collection_id", "source", "source_workspace", "source_kind", "method", "seed", + "wangp_processor_settings", + } + assert "actor" in schema["excluded"] + assert "filesystem paths" in schema["excluded"] diff --git a/ui/e2e/helpers/apiRoutes.ts b/ui/e2e/helpers/apiRoutes.ts index c4669106a..55f554e11 100644 --- a/ui/e2e/helpers/apiRoutes.ts +++ b/ui/e2e/helpers/apiRoutes.ts @@ -541,14 +541,33 @@ export async function installApiRoutes(page: Page, options: ApiRouteOptions = {} })) return } - if (method === 'POST' && pathname === '/api/v1/tools/upscale') { + if (method === 'POST' && pathname === '/api/v1/generation/commands') { + const body = JSON.parse(request.postData() || '{}') as Record upscaleSubmitted = true upscaleStatusCalls = 0 upscaleCancelRequested = false await route.fulfill(json({ - job_id: 'tool-upscale-e2e', - task_id: 'task-generation-tool-upscale-e2e', - root_task_id: 'task-generation-tool-upscale-e2e', + receipt: { + version: 1, + commandId: body.intent_id, + operation: 'tools.upscale', + status: 'queued', + entities: [], + artifacts: [], + taskIds: ['task-generation-tool-upscale-e2e'], + pipelineIds: [], + result: { + job_id: 'tool-upscale-e2e', + task_id: 'task-generation-tool-upscale-e2e', + root_task_id: 'task-generation-tool-upscale-e2e', + workspace: 'default', + status: 'queued', + }, + commandVersion: 2, + contentFingerprint: 'a'.repeat(64), + fingerprintVersion: 2, + }, + replayed: false, })) return } diff --git a/ui/e2e/specs/tools-background-removal.spec.ts b/ui/e2e/specs/tools-background-removal.spec.ts index 15d909208..63270557e 100644 --- a/ui/e2e/specs/tools-background-removal.spec.ts +++ b/ui/e2e/specs/tools-background-removal.spec.ts @@ -122,7 +122,8 @@ test('keeps a tool failure visible in the activity card', async ({ page }) => { test('runs the shared Upscale action from an image and publishes a derived asset', async ({ page }) => { const session = await gotoApp(page, { upscaleMode: 'complete' }) - const submissions = collectRequests(page, '/api/v1/tools/upscale') + const submissions = collectRequests(page, '/api/v1/generation/commands') + const legacySubmissions = collectRequests(page, '/api/v1/tools/upscale') const statuses = collectRequests(page, '/api/v1/status/tool-upscale-e2e') try { @@ -134,18 +135,51 @@ test('runs the shared Upscale action from an image and publishes a derived asset await expect(page.getByRole('img', { name: 'hero.png', exact: true })).toBeVisible() const imageRun = page.getByRole('button', { name: 'Upscale Image', exact: true }) await expect(imageRun).toBeEnabled() + const acknowledgement = page.waitForResponse(response => ( + new URL(response.url()).pathname === '/api/v1/generation/commands' + && response.request().method() === 'POST' + )) await imageRun.click() await expect.poll(() => submissions.length).toBe(1) const payload = JSON.parse(submissions[0].postData() || '{}') as Record expect(payload).toMatchObject({ - source: 'hero.png', + version: 2, + operation: 'tools.upscale', + intent_id: expect.any(String), + }) + const input = payload.input as Record + const params = input.params as Record + expect(input).toMatchObject({ workspace: 'default' }) + expect(params).toMatchObject({ + source: 'asset-hero', source_kind: 'image', - asset_id: 'asset-hero', source_workspace: 'default', - workspace: 'default', }) + expect(payload.asset_id).toBeUndefined() + expect(payload.source).toBeUndefined() expect(payload.video_path).toBeUndefined() + const response = await acknowledgement + expect(response.status()).toBe(200) + const envelope = await response.json() as Record + expect(envelope.replayed).toBe(false) + expect(envelope.receipt).toMatchObject({ + version: 1, + commandId: payload.intent_id, + operation: 'tools.upscale', + status: 'queued', + taskIds: ['task-generation-tool-upscale-e2e'], + result: { + job_id: 'tool-upscale-e2e', + task_id: 'task-generation-tool-upscale-e2e', + workspace: 'default', + status: 'queued', + }, + commandVersion: 2, + fingerprintVersion: 2, + contentFingerprint: expect.stringMatching(/^[a-f0-9]{64}$/), + }) + expect(legacySubmissions).toHaveLength(0) await expect(page.getByText('Queued...', { exact: true }).first()).toBeVisible() await expect.poll(() => statuses.length, { timeout: 10_000 }).toBeGreaterThanOrEqual(2) diff --git a/ui/src/api/generationCommandClient.ts b/ui/src/api/generationCommandClient.ts new file mode 100644 index 000000000..8b9f42b83 --- /dev/null +++ b/ui/src/api/generationCommandClient.ts @@ -0,0 +1,727 @@ +import { BASE } from './http' +import { stableSerialize } from '../lib/commandContract' +import type { GenerationSubmissionContext } from '../features/studio/generationProvenance' + +/** + * Durable client-side transport shared by the typed Studio generation + * operations. A command-specific module owns its closed schema and passes + * only a detached validator here; this module owns persistence, ACK ordering, + * retries and receipt validation. + */ +export interface GenerationCommandInput { + workspace: string + [key: string]: unknown +} + +export interface GenerationCommandLike { + version: number + operation: string + intent_id: string + input: { workspace: string } +} + +export interface GenerationTaskResult { + job_id: string + task_id: string + workspace: string + status: 'queued' + root_task_id?: string | null +} + +export interface GenerationReceiptLike { + version: 1 + commandId: string + operation: string + status: 'queued' + entities: unknown[] + artifacts: unknown[] + taskIds: string[] + pipelineIds: string[] + result: GenerationTaskResult + replayed?: boolean + commandVersion?: 2 + contentFingerprint?: string + fingerprintVersion?: 2 +} + +export interface GenerationCommandErrorOptions { + status?: number + uncertain?: boolean + code?: string +} + +export class GenerationCommandError extends Error { + readonly intentId: string + readonly workspace: string + readonly status?: number + readonly uncertain: boolean + readonly code: string + + constructor( + message: string, + intentId: string, + workspace: string, + options: GenerationCommandErrorOptions = {}, + ) { + super(message) + this.name = 'GenerationCommandError' + this.intentId = intentId + this.workspace = workspace + this.status = options.status + this.uncertain = options.uncertain ?? false + this.code = options.code ?? 'generation_command_failed' + } +} + +type CommandErrorConstructor = new ( + message: string, + intentId: string, + workspace: string, + options?: GenerationCommandErrorOptions, +) => GenerationCommandError + +export interface GenerationCommandClientConfig< + Command extends GenerationCommandLike, + Receipt extends GenerationReceiptLike, +> { + /** Stable storage namespace. It must include the versioned operation family. */ + storagePrefix: string + contextStoragePrefix: string + pendingChangedEvent: string + operation: string + label: string + /** A v1 fallback keeps legacy receipt reads compatible when no hint exists. */ + receiptFallbackVersion?: number + detach: (value: unknown) => Command + errorClass?: CommandErrorConstructor + /** Build a receipt type without exposing a second transport implementation. */ + castReceipt?: (value: GenerationReceiptLike) => Receipt +} + +export interface SubmitGenerationCommandOptions { + /** UI attribution is transport metadata; it never enters the command hash. */ + submissionContext?: GenerationSubmissionContext + /** Runs after the durable pending hint and before the network POST. */ + onSnapshotReady?: (snapshot: Command) => void | Promise +} + +type StoredSubmissionContext = Pick + +interface ReceiptEnvelope { + receipt: unknown + replayed?: boolean + malformed?: boolean +} + +interface ReceiptContext { + version: number + intent_id: string + operation: string + input: Pick +} + +const MAX_INTENT_LENGTH = 160 +const MAX_WORKSPACE_LENGTH = 240 +const MAX_SUBMISSION_CONTEXT_ID_LENGTH = 200 +const SUBMISSION_ACTORS = new Set(['user', 'wizard', 'system', 'unknown']) + +function isRecord(value: unknown): value is Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false + const prototype = Object.getPrototypeOf(value) + return prototype === Object.prototype || prototype === null +} + +function errorFor( + config: GenerationCommandClientConfig, + message: string, + command: Pick & { input: Pick }, + options: GenerationCommandErrorOptions = {}, +): GenerationCommandError { + const ErrorClass = config.errorClass || GenerationCommandError + return new ErrorClass(message, command.intent_id, command.input.workspace, options) +} + +function requiredText(value: unknown, field: string, maximum: number): string { + if (typeof value !== 'string' || !value.trim()) throw new Error(`${field} must be a non-blank string`) + if (value.length > maximum) throw new Error(`${field} is too long`) + return value +} + +function storage(): Storage { + if (typeof globalThis.localStorage === 'undefined') { + throw new Error('localStorage is unavailable; command admission cannot be made safely') + } + return globalThis.localStorage +} + +function notifyPendingChanged(eventName: string): void { + if (typeof window === 'undefined' || typeof Event === 'undefined') return + window.dispatchEvent(new Event(eventName)) +} + +function contextPart(value: unknown, field: string): string | undefined { + if (value === undefined || value === null || value === '') return undefined + if (typeof value !== 'string' || !value.trim() || value.trim() !== value + || value.length > MAX_SUBMISSION_CONTEXT_ID_LENGTH) { + throw new Error(field + ' must be an exact non-blank string of at most 200 characters') + } + return value +} + +function validateSubmissionContext(context: GenerationSubmissionContext | undefined): StoredSubmissionContext | undefined { + if (!context) return undefined + if (!SUBMISSION_ACTORS.has(context.actor)) throw new Error('submissionContext.actor must be a known actor') + const workflowId = contextPart(context.workflowId, 'submissionContext.workflowId') + const runId = contextPart(context.runId, 'submissionContext.runId') + return { + actor: context.actor, + ...(workflowId !== undefined ? { workflowId } : {}), + ...(runId !== undefined ? { runId } : {}), + } +} + +function receiptError( + config: GenerationCommandClientConfig, + command: ReceiptContext, + message = 'Receipt could not be verified', +): GenerationCommandError { + return errorFor(config, `${message} for ${command.intent_id}`, command, { + status: 200, + uncertain: true, + code: 'invalid_receipt', + }) +} + +function pendingKey( + config: GenerationCommandClientConfig, + intentId: string, +): string { + return config.storagePrefix + intentId +} + +function pendingContextKey( + config: GenerationCommandClientConfig, + intentId: string, +): string { + return config.contextStoragePrefix + intentId +} + +function invalidStoredCommand( + config: GenerationCommandClientConfig, + intentId: string, +): GenerationCommandError { + return errorFor(config, `Stored ${config.label} command ${intentId} is invalid`, { + intent_id: intentId, + input: { workspace: '' }, + }, { code: 'invalid_pending_command' }) +} + +function readPending( + config: GenerationCommandClientConfig, + intentId: string, +): Command | null { + const raw = storage().getItem(pendingKey(config, intentId)) + if (raw == null) return null + try { + const command = config.detach(JSON.parse(raw)) + if (command.intent_id !== intentId) throw new Error('intent_id does not match storage key') + return command + } catch { + throw invalidStoredCommand(config, intentId) + } +} + +const STORED_CONTEXT_ENVELOPE_FIELDS = new Set(['version', 'intent_id', 'workspace', 'context']) +const STORED_CONTEXT_FIELDS = new Set(['actor', 'workflowId', 'runId']) + +function invalidStoredContext( + config: GenerationCommandClientConfig, + intentId: string, +): GenerationCommandError { + return errorFor(config, `Stored ${config.label} context ${intentId} is invalid`, { + intent_id: intentId, + input: { workspace: '' }, + }, { code: 'invalid_pending_context' }) +} + +function normalizeStoredContext(value: unknown): StoredSubmissionContext { + if (!isRecord(value)) throw new Error('context must be an object') + for (const key of Object.keys(value)) { + if (!STORED_CONTEXT_FIELDS.has(key)) throw new Error('context contains an unsupported field') + } + if (typeof value.actor !== 'string' || !SUBMISSION_ACTORS.has(value.actor)) throw new Error('context.actor is invalid') + const workflowId = contextPart(value.workflowId, 'context.workflowId') + const runId = contextPart(value.runId, 'context.runId') + return { + actor: value.actor as StoredSubmissionContext['actor'], + ...(workflowId !== undefined ? { workflowId } : {}), + ...(runId !== undefined ? { runId } : {}), + } +} + +function readPendingContext( + config: GenerationCommandClientConfig, + command: Command, +): StoredSubmissionContext | null { + const raw = storage().getItem(pendingContextKey(config, command.intent_id)) + if (raw == null) return null + try { + const value: unknown = JSON.parse(raw) + if (!isRecord(value) + || Object.keys(value).some(key => !STORED_CONTEXT_ENVELOPE_FIELDS.has(key)) + || value.version !== 1 + || value.intent_id !== command.intent_id + || value.workspace !== command.input.workspace) throw new Error('context envelope does not match command') + return normalizeStoredContext(value.context) + } catch { + throw invalidStoredContext(config, command.intent_id) + } +} + +function persistPendingContext( + config: GenerationCommandClientConfig, + command: Command, + context: StoredSubmissionContext | undefined, +): void { + const key = pendingContextKey(config, command.intent_id) + if (!context) { + if (storage().getItem(key) != null) storage().removeItem(key) + return + } + storage().setItem(key, stableSerialize({ + version: 1, + intent_id: command.intent_id, + workspace: command.input.workspace, + context, + })) +} + +function sameSubmissionContext(left: StoredSubmissionContext, right: StoredSubmissionContext): boolean { + return stableSerialize(left) === stableSerialize(right) +} + +function sameCommand(left: Command, right: Command): boolean { + return stableSerialize(left) === stableSerialize(right) +} + +function retainPending( + config: GenerationCommandClientConfig, + command: Command, + done = false, +): boolean { + const existing = readPending(config, command.intent_id) + if (existing && !sameCommand(existing, command)) { + throw errorFor(config, `intent_id ${command.intent_id} is already pending with a different command`, command, { + code: 'intent_conflict', + }) + } + const key = pendingKey(config, command.intent_id) + if (done) { + const current = readPending(config, command.intent_id) + if (current && sameCommand(current, command)) { + persistPendingContext(config, command, undefined) + storage().removeItem(key) + } else if (!current) { + persistPendingContext(config, command, undefined) + } + } else { + storage().setItem(key, stableSerialize(command)) + } + notifyPendingChanged(config.pendingChangedEvent) + return existing != null +} + +function forgetPending( + config: GenerationCommandClientConfig, + command: Command, +): void { + try { retainPending(config, command, true) } catch { /* cleanup is best effort after a confirmed receipt */ } +} + +function errorDetail(value: unknown): string | undefined { + if (typeof value === 'string' && value.trim()) return value + if (!isRecord(value)) return undefined + const detail = value.detail + if (typeof detail === 'string' && detail.trim()) return detail + if (isRecord(detail) && typeof detail.message === 'string' && detail.message.trim()) return detail.message + if (typeof value.message === 'string' && value.message.trim()) return value.message + return undefined +} + +async function responseError( + config: GenerationCommandClientConfig, + response: Response, + command: Pick & { input: Pick }, +): Promise { + const payload = await response.json().catch(() => undefined) + return errorFor(config, errorDetail(payload) || `${config.label} command failed (${response.status})`, command, { + status: response.status, + uncertain: response.status >= 500, + code: 'http_error', + }) +} + +function unwrapReceipt(value: unknown): ReceiptEnvelope { + if (isRecord(value) && 'receipt' in value) { + if ('replayed' in value && typeof value.replayed !== 'boolean') return { receipt: value.receipt, malformed: true } + return { + receipt: value.receipt, + replayed: typeof value.replayed === 'boolean' ? value.replayed : undefined, + } + } + return { receipt: value } +} + +function receiptReplay( + config: GenerationCommandClientConfig, + value: Record, + command: ReceiptContext, + fallback?: boolean, +): boolean | undefined { + if ('replayed' in value && typeof value.replayed !== 'boolean') throw receiptError(config, command) + return typeof value.replayed === 'boolean' ? value.replayed : fallback +} + +function receiptV2Metadata( + config: GenerationCommandClientConfig, + value: Record, + command: ReceiptContext, +): Pick { + const fields = ['commandVersion', 'contentFingerprint', 'fingerprintVersion'] as const + const present = fields.filter(field => field in value) + if (present.length !== 0 && present.length !== fields.length) { + throw receiptError(config, command, 'Receipt fingerprint metadata is incomplete') + } + if (present.length === 0) { + if (command.version === 2) throw receiptError(config, command, 'The v2 receipt is missing its content fingerprint') + return {} + } + if (value.commandVersion !== 2 || value.fingerprintVersion !== 2 + || typeof value.contentFingerprint !== 'string' + || !/^[a-f0-9]{64}$/.test(value.contentFingerprint)) { + throw receiptError(config, command, 'Receipt fingerprint metadata is invalid') + } + return { commandVersion: 2, contentFingerprint: value.contentFingerprint, fingerprintVersion: 2 } +} + +function validTaskResult(value: unknown, workspace: string, taskIds: unknown[]): value is GenerationTaskResult { + return isRecord(value) + && typeof value.job_id === 'string' && value.job_id.length > 0 + && typeof value.task_id === 'string' && value.task_id.length > 0 + && taskIds.length === 1 && taskIds[0] === value.task_id + && value.workspace === workspace && value.status === 'queued' +} + +function validateReceipt( + config: GenerationCommandClientConfig, + value: unknown, + command: ReceiptContext, + replayed?: boolean, +): Receipt { + if (!isRecord(value)) throw receiptError(config, command) + const outerReplayed = receiptReplay(config, value, command, replayed) + const metadata = receiptV2Metadata(config, value, command) + const taskIds = value.taskIds + if (value.version !== 1 + || value.commandId !== command.intent_id + || value.operation !== command.operation + || value.status !== 'queued' + || !Array.isArray(value.entities) + || !Array.isArray(value.artifacts) + || !Array.isArray(taskIds) + || !Array.isArray(value.pipelineIds) + || !validTaskResult(value.result, command.input.workspace, taskIds)) { + throw receiptError(config, command) + } + const receipt: GenerationReceiptLike = { + version: 1, + commandId: command.intent_id, + operation: command.operation, + status: 'queued', + entities: JSON.parse(stableSerialize(value.entities)) as unknown[], + artifacts: JSON.parse(stableSerialize(value.artifacts)) as unknown[], + taskIds: [...taskIds] as string[], + pipelineIds: [...value.pipelineIds] as string[], + result: JSON.parse(stableSerialize(value.result)) as GenerationTaskResult, + ...metadata, + } + if (outerReplayed !== undefined) receipt.replayed = outerReplayed + return config.castReceipt ? config.castReceipt(receipt) : receipt as Receipt +} + +function requestedContext( + config: GenerationCommandClientConfig, + snapshot: Command, + context: GenerationSubmissionContext | undefined, +): StoredSubmissionContext | undefined { + try { + return validateSubmissionContext(context) + } catch (error) { + throw errorFor(config, error instanceof Error ? error.message : 'The submission context is invalid', snapshot, { + code: 'invalid_submission_context', + }) + } +} + +function assertContextMatches( + config: GenerationCommandClientConfig, + snapshot: Command, + stored: StoredSubmissionContext | null, + requested: StoredSubmissionContext | undefined, +): void { + if (stored && requested && !sameSubmissionContext(stored, requested)) { + throw errorFor(config, `intent_id ${snapshot.intent_id} is already attributed to a different UI context`, snapshot, { + code: 'submission_context_conflict', uncertain: true, + }) + } +} + +function preparePendingSubmission( + config: GenerationCommandClientConfig, + snapshot: Command, + requested: StoredSubmissionContext | undefined, +): { recovering: boolean; submissionContext?: StoredSubmissionContext } { + let recovering = false + try { + const existing = readPending(config, snapshot.intent_id) + if (existing && !sameCommand(existing, snapshot)) { + throw errorFor(config, `intent_id ${snapshot.intent_id} is already pending with a different command`, snapshot, { + code: 'intent_conflict', + }) + } + recovering = existing !== null + if (!recovering) persistPendingContext(config, snapshot, requested) + recovering = retainPending(config, snapshot) + const stored = recovering ? readPendingContext(config, snapshot) : null + assertContextMatches(config, snapshot, stored, requested) + if (recovering && !stored && requested) persistPendingContext(config, snapshot, requested) + return { recovering, submissionContext: stored || requested } + } catch (error) { + if (error instanceof GenerationCommandError) throw error + throw errorFor(config, error instanceof Error ? error.message : `Could not persist ${config.label} command`, snapshot, { + code: 'pending_storage_failed', uncertain: recovering, + }) + } +} + +async function presentSnapshot( + config: GenerationCommandClientConfig, + snapshot: Command, + recovering: boolean, + hook: SubmitGenerationCommandOptions['onSnapshotReady'], +): Promise { + if (!hook) return + try { + await hook(config.detach(snapshot)) + } catch (error) { + if (!recovering) forgetPending(config, snapshot) + throw errorFor(config, error instanceof Error ? error.message : `${config.label} command could not be presented`, snapshot, { + code: 'snapshot_hook_failed', + }) + } +} + +async function postCommand( + config: GenerationCommandClientConfig, + command: Command, + submissionContext?: StoredSubmissionContext, +): Promise { + let response: Response + try { + const headers: Record = { 'Content-Type': 'application/json' } + if (submissionContext) { + headers['X-Hocus-UI-Surface'] = submissionContext.actor === 'wizard' ? 'wizard' : 'studio' + const context: Record = {} + const workflowId = contextPart(submissionContext.workflowId, 'submissionContext.workflowId') + const runId = contextPart(submissionContext.runId, 'submissionContext.runId') + if (workflowId !== undefined) context.workflowId = workflowId + if (runId !== undefined) context.runId = runId + if (Object.keys(context).length > 0) headers['X-Hocus-UI-Context'] = JSON.stringify(context) + } + response = await fetch(`${BASE}/api/v1/generation/commands`, { + method: 'POST', + headers, + body: JSON.stringify(command), + }) + } catch (error) { + throw errorFor(config, error instanceof Error ? error.message : `${config.label} request failed`, command, { + uncertain: true, code: 'transport_uncertain', + }) + } + if (!response.ok) throw await responseError(config, response, command) + try { + return await response.json() + } catch { + throw errorFor(config, `${config.label} response could not be decoded for ${command.intent_id}`, command, { + status: response.status, uncertain: true, code: 'invalid_response', + }) + } +} + +function receiptCommandContext( + config: GenerationCommandClientConfig, + intentId: string, + workspace: string, +): ReceiptContext { + const fallback: ReceiptContext = { + version: config.receiptFallbackVersion ?? 1, + intent_id: intentId, + operation: config.operation, + input: { workspace }, + } + try { + const pending = readPending(config, intentId) + return pending && pending.input.workspace === workspace ? pending : fallback + } catch { + return fallback + } +} + +async function requestReceipt( + config: GenerationCommandClientConfig, + workspace: string, + intentId: string, +): Promise { + const command = { intent_id: intentId, input: { workspace } } + try { + const response = await fetch(`${BASE}/api/v1/generation/commands/receipt?workspace=${encodeURIComponent(workspace)}&intent_id=${encodeURIComponent(intentId)}`) + if (!response.ok) throw await responseError(config, response, command) + return response + } catch (error) { + if (error instanceof GenerationCommandError) throw error + throw errorFor(config, error instanceof Error ? error.message : `${config.label} receipt request failed`, command, { + uncertain: true, code: 'transport_uncertain', + }) + } +} + +async function decodeReceiptResponse( + config: GenerationCommandClientConfig, + response: Response, + command: ReceiptContext, +): Promise { + try { + const payload = unwrapReceipt(await response.json()) + if (payload.malformed) throw receiptError(config, command) + return validateReceipt(config, payload.receipt, command, payload.replayed) + } catch (error) { + if (error instanceof GenerationCommandError) throw error + throw errorFor(config, error instanceof Error ? error.message : `${config.label} receipt could not be verified`, command, { + status: 200, uncertain: true, code: 'invalid_receipt', + }) + } +} + +function clearPendingReceipt( + config: GenerationCommandClientConfig, + intentId: string, + workspace: string, +): void { + try { + const pending = readPending(config, intentId) + if (pending && pending.input.workspace === workspace) forgetPending(config, pending) + } catch { /* A valid server receipt remains authoritative. */ } +} + +function isDefinitiveClientError(error: GenerationCommandError): boolean { + return error.status !== undefined && error.status >= 400 && error.status < 500 +} + +function normalizeFailure( + config: GenerationCommandClientConfig, + error: unknown, + snapshot: Command, + recovering: boolean, +): GenerationCommandError { + const commandError = error instanceof GenerationCommandError + ? error + : errorFor(config, error instanceof Error ? error.message : `${config.label} command failed`, snapshot, { + uncertain: true, code: 'unknown_failure', + }) + if (!recovering && isDefinitiveClientError(commandError)) forgetPending(config, snapshot) + return errorFor(config, commandError.message, snapshot, { + status: commandError.status, + uncertain: commandError.uncertain || recovering, + code: commandError.code, + }) +} + +export interface GenerationCommandClient { + newIntentId: () => string + pendingCommands: (workspace?: string) => Command[] + pendingCommand: (intentId: string, workspace?: string) => Command | null + submit: (command: Command, options?: SubmitGenerationCommandOptions) => Promise + fetchReceipt: (workspace: string, intentId: string) => Promise + subscribe: (callback: () => void) => () => void +} + +export function createGenerationCommandClient( + config: GenerationCommandClientConfig, +): GenerationCommandClient { + const newIntentId = () => globalThis.crypto?.randomUUID?.() + || `${config.operation.replace(/[^a-z0-9]+/gi, '-')}-${Date.now()}-${Math.random().toString(36).slice(2)}` + + const pendingCommands = (workspace?: string): Command[] => { + const pending: Command[] = [] + const area = storage() + for (let index = 0; index < area.length; index += 1) { + const key = area.key(index) + if (!key?.startsWith(config.storagePrefix)) continue + const intentId = key.slice(config.storagePrefix.length) + let command: Command | null + try { + command = readPending(config, intentId) + } catch (error) { + // One damaged hint must not hide unrelated recoverable admissions. + // Keep its bytes and let direct lookup/retry report the corruption. + if (error instanceof GenerationCommandError && error.code === 'invalid_pending_command') continue + throw error + } + if (!command || (workspace !== undefined && command.input.workspace !== workspace)) continue + pending.push(command) + } + return pending.sort((left, right) => left.intent_id.localeCompare(right.intent_id)) + } + + const pendingCommand = (intentId: string, workspace?: string): Command | null => { + const command = readPending(config, intentId) + if (!command || (workspace !== undefined && command.input.workspace !== workspace)) return null + return command + } + + const submit = async (command: Command, options: SubmitGenerationCommandOptions = {}): Promise => { + const snapshot = config.detach(command) + const context = requestedContext(config, snapshot, options.submissionContext) + const prepared = preparePendingSubmission(config, snapshot, context) + await presentSnapshot(config, snapshot, prepared.recovering, options.onSnapshotReady) + try { + const envelope = unwrapReceipt(await postCommand(config, snapshot, prepared.submissionContext)) + if (envelope.malformed) throw receiptError(config, snapshot) + const receipt = validateReceipt(config, envelope.receipt, snapshot, envelope.replayed) + forgetPending(config, snapshot) + return receipt + } catch (error) { + throw normalizeFailure(config, error, snapshot, prepared.recovering) + } + } + + const fetchReceipt = async (workspace: string, intentId: string): Promise => { + requiredText(workspace, 'workspace', MAX_WORKSPACE_LENGTH) + requiredText(intentId, 'intent_id', MAX_INTENT_LENGTH) + const context = receiptCommandContext(config, intentId, workspace) + const receipt = await decodeReceiptResponse(config, await requestReceipt(config, workspace, intentId), context) + clearPendingReceipt(config, intentId, workspace) + return receipt + } + + const subscribe = (callback: () => void): (() => void) => { + window.addEventListener(config.pendingChangedEvent, callback) + window.addEventListener('storage', callback) + return () => { + window.removeEventListener(config.pendingChangedEvent, callback) + window.removeEventListener('storage', callback) + } + } + + return { newIntentId, pendingCommands, pendingCommand, submit, fetchReceipt, subscribe } +} diff --git a/ui/src/api/imageCommandCatalog.json b/ui/src/api/imageCommandCatalog.json index 4c763ce0f..bdf6e2495 100644 --- a/ui/src/api/imageCommandCatalog.json +++ b/ui/src/api/imageCommandCatalog.json @@ -1646,7 +1646,7 @@ "version": 1, "domain": "studio", "mutation": false, - "description": "Read an immutable image admission and its current canonical task in the exact original output workspace.", + "description": "Read an immutable generation admission and its current canonical task in the exact original output workspace.", "inputSchema": { "type": "object", "additionalProperties": false, diff --git a/ui/src/api/imageGenerationCommands.ts b/ui/src/api/imageGenerationCommands.ts index 03f112a80..9babfb38b 100644 --- a/ui/src/api/imageGenerationCommands.ts +++ b/ui/src/api/imageGenerationCommands.ts @@ -1,6 +1,12 @@ -import { BASE } from './http' import { stableSerialize } from '../lib/commandContract' -import type { GenerationSubmissionContext } from '../features/studio/generationProvenance' +import { + createGenerationCommandClient, + GenerationCommandError, + type GenerationReceiptLike, + type GenerationTaskResult, + type GenerationCommandErrorOptions, + type SubmitGenerationCommandOptions, +} from './generationCommandClient' import { assertStudioImageGenerationCommand, detachedStudioImageGenerationCommand, @@ -18,7 +24,7 @@ export type { StudioImageParams, } from '../features/studio/generationSpec' -/** The first shared generation vertical deliberately exposes image only. */ +/** The legacy image command remains supported beside the typed Studio command. */ export const IMAGE_GENERATION_OPERATION = 'generation.image' as const export const IMAGE_GENERATION_SCHEMA_VERSION = 1 as const @@ -45,69 +51,33 @@ export interface ImageGenerationCommandV1 { export type ImageGenerationCommandV2 = StudioImageGenerationCommand export type ImageGenerationCommand = ImageGenerationCommandV1 | ImageGenerationCommandV2 -export interface ImageGenerationTaskResult { - job_id: string - task_id: string - workspace: string - status: 'queued' - root_task_id?: string | null -} +export type ImageGenerationTaskResult = GenerationTaskResult -export interface ImageGenerationReceipt { +export interface ImageGenerationReceipt extends GenerationReceiptLike { version: typeof IMAGE_GENERATION_SCHEMA_VERSION - commandId: string operation: typeof IMAGE_GENERATION_OPERATION - status: 'queued' - entities: unknown[] - artifacts: unknown[] - taskIds: string[] - pipelineIds: string[] result: ImageGenerationTaskResult - replayed?: boolean - commandVersion?: 2 - contentFingerprint?: string - fingerprintVersion?: 2 } -/** - * Recovery metadata is deliberately smaller than the caller context. It is - * stored beside the immutable command hint and only drives declared UI - * attribution headers; it is never part of the command or its fingerprint. - */ -type StoredSubmissionContext = Pick - -export class ImageGenerationCommandError extends Error { - readonly intentId: string - readonly workspace: string - readonly status?: number - readonly uncertain: boolean - readonly code: string - +export class ImageGenerationCommandError extends GenerationCommandError { constructor( message: string, intentId: string, workspace: string, - options: { status?: number; uncertain?: boolean; code?: string } = {}, + options: GenerationCommandErrorOptions = {}, ) { - super(message) + super(message, intentId, workspace, { + ...options, + code: options.code ?? 'image_generation_command_failed', + }) this.name = 'ImageGenerationCommandError' - this.intentId = intentId - this.workspace = workspace - this.status = options.status - this.uncertain = options.uncertain ?? false - this.code = options.code ?? 'image_generation_command_failed' } } -const PENDING_KEY_PREFIX = 'hocuspocus.generation.image-commands.v1:' -const PENDING_CONTEXT_KEY_PREFIX = 'hocuspocus.generation.image-command-context.v1:' -const PENDING_CHANGED_EVENT = 'hocuspocus:generation-image-commands-changed' const MAX_INTENT_LENGTH = 160 const MAX_ID_LENGTH = 240 const MAX_PROMPT_LENGTH = 200_000 const MAX_RESOLUTION_LENGTH = 128 -const MAX_SUBMISSION_CONTEXT_ID_LENGTH = 200 -const SUBMISSION_ACTORS = new Set(['user', 'wizard', 'system', 'unknown']) const INPUT_FIELDS = new Set([ 'workspace', @@ -195,186 +165,32 @@ export function assertImageGenerationCommandV1(value: unknown): asserts value is assertImageGenerationInput(value.input) } -function detachedCommand(value: unknown): ImageGenerationCommand { +function detachedImageCommand(value: unknown): ImageGenerationCommand { if (isRecord(value) && value.version === 2) { assertStudioImageGenerationCommand(value) return detachedStudioImageGenerationCommand(value) } assertImageGenerationCommandV1(value) - // stableSerialize validates the JSON boundary and gives the retry an - // immutable value-level snapshot. It never adds native defaults to input. - return JSON.parse(stableSerialize(value)) as ImageGenerationCommand -} - -function storage(): Storage { - if (typeof globalThis.localStorage === 'undefined') { - throw new Error('localStorage is unavailable; command admission cannot be made safely') - } - return globalThis.localStorage -} - -function notifyPendingChanged(): void { - if (typeof window === 'undefined' || typeof Event === 'undefined') return - window.dispatchEvent(new Event(PENDING_CHANGED_EVENT)) -} - -function pendingKey(intentId: string): string { - return PENDING_KEY_PREFIX + intentId -} - -function invalidStoredCommand(intentId: string): ImageGenerationCommandError { - return new ImageGenerationCommandError( - `Stored image generation command ${intentId} is invalid`, - intentId, - '', - { code: 'invalid_pending_command' }, - ) -} - -function readPending(intentId: string): ImageGenerationCommand | null { - const raw = storage().getItem(pendingKey(intentId)) - if (raw == null) return null - try { - const value: unknown = JSON.parse(raw) - const command = detachedCommand(value) - if (command.intent_id !== intentId) throw new Error('intent_id does not match storage key') - return command - } catch { - throw invalidStoredCommand(intentId) - } -} - -const STORED_CONTEXT_ENVELOPE_FIELDS = new Set(['version', 'intent_id', 'workspace', 'context']) -const STORED_CONTEXT_FIELDS = new Set(['actor', 'workflowId', 'runId']) - -function pendingContextKey(intentId: string): string { - return PENDING_CONTEXT_KEY_PREFIX + intentId -} - -function invalidStoredContext(intentId: string): ImageGenerationCommandError { - return new ImageGenerationCommandError( - `Stored image generation context ${intentId} is invalid`, - intentId, - '', - { code: 'invalid_pending_context' }, - ) -} - -function normalizeStoredSubmissionContext(value: unknown): StoredSubmissionContext { - if (!isRecord(value)) throw new Error('context must be an object') - for (const key of Object.keys(value)) { - if (!STORED_CONTEXT_FIELDS.has(key)) throw new Error('context contains an unsupported field') - } - if (typeof value.actor !== 'string' || !SUBMISSION_ACTORS.has(value.actor)) { - throw new Error('context.actor is invalid') - } - const workflowId = submissionContextPart(value.workflowId, 'context.workflowId') - const runId = submissionContextPart(value.runId, 'context.runId') - return { - actor: value.actor as StoredSubmissionContext['actor'], - ...(workflowId !== undefined ? { workflowId } : {}), - ...(runId !== undefined ? { runId } : {}), - } -} - -function readPendingContext(command: ImageGenerationCommand): StoredSubmissionContext | null { - const raw = storage().getItem(pendingContextKey(command.intent_id)) - if (raw == null) return null - try { - const value: unknown = JSON.parse(raw) - if (!isRecord(value) - || Object.keys(value).some(key => !STORED_CONTEXT_ENVELOPE_FIELDS.has(key)) - || value.version !== 1 - || value.intent_id !== command.intent_id - || value.workspace !== command.input.workspace) { - throw new Error('context envelope does not match its command') - } - return normalizeStoredSubmissionContext(value.context) - } catch { - throw invalidStoredContext(command.intent_id) - } -} - -function persistPendingContext( - command: ImageGenerationCommand, - context: StoredSubmissionContext | undefined, -): void { - const key = pendingContextKey(command.intent_id) - if (!context) { - // Avoid turning a no-op cleanup into a storage failure. This also keeps - // the legacy command-only path compatible with callers whose storage - // implementation rejects removeItem even when the key is absent. - if (storage().getItem(key) != null) storage().removeItem(key) - return - } - storage().setItem(key, stableSerialize({ - version: 1, - intent_id: command.intent_id, - workspace: command.input.workspace, - context, - })) -} - -function sameSubmissionContext( - left: StoredSubmissionContext, - right: StoredSubmissionContext, -): boolean { - return stableSerialize(left) === stableSerialize(right) -} - -function sameCommand(left: ImageGenerationCommand, right: ImageGenerationCommand): boolean { - return stableSerialize(left) === stableSerialize(right) -} - -function retainPending(command: ImageGenerationCommand, done = false): boolean { - const existing = readPending(command.intent_id) - if (existing && !sameCommand(existing, command)) { - throw new ImageGenerationCommandError( - `intent_id ${command.intent_id} is already pending with a different command`, - command.intent_id, - command.input.workspace, - { code: 'intent_conflict' }, - ) - } - const key = pendingKey(command.intent_id) - if (done) { - // A different tab may have replaced the value. Never erase that command - // while cleaning up a receipt for this one. - const current = readPending(command.intent_id) - if (current && sameCommand(current, command)) { - // Clear the sidecar first. If storage cleanup fails, retain the command - // hint so a confirmed result remains recoverable as before this sidecar - // existed. - persistPendingContext(command, undefined) - storage().removeItem(key) - } else if (!current) { - // The command may already have been removed by another tab after its - // receipt was confirmed. Its context key is still scoped by intent and - // can be cleaned without touching a replacement command. - persistPendingContext(command, undefined) - } - } else { - storage().setItem(key, stableSerialize(command)) - } - notifyPendingChanged() - return existing != null + return JSON.parse(stableSerialize(value)) as ImageGenerationCommandV1 } -function forgetPending(command: ImageGenerationCommand): void { - try { retainPending(command, true) } catch { /* A cleanup failure must not hide a confirmed server result. */ } -} +const imageCommandClient = createGenerationCommandClient({ + storagePrefix: 'hocuspocus.generation.image-commands.v1:', + contextStoragePrefix: 'hocuspocus.generation.image-command-context.v1:', + pendingChangedEvent: 'hocuspocus:generation-image-commands-changed', + operation: IMAGE_GENERATION_OPERATION, + label: 'Image generation', + receiptFallbackVersion: IMAGE_GENERATION_SCHEMA_VERSION, + detach: detachedImageCommand, + errorClass: ImageGenerationCommandError, + castReceipt: value => value as ImageGenerationReceipt, +}) -export function newImageGenerationIntentId(): string { - return globalThis.crypto?.randomUUID?.() - || `image-${Date.now()}-${Math.random().toString(36).slice(2)}` -} - -/** Require the caller to choose the intention; this function never invents one. */ export function createImageGenerationCommand( intentId: string, input: ImageGenerationInput, ): ImageGenerationCommandV1 { - return detachedCommand({ + return detachedImageCommand({ version: IMAGE_GENERATION_SCHEMA_VERSION, operation: IMAGE_GENERATION_OPERATION, intent_id: intentId, @@ -382,525 +198,16 @@ export function createImageGenerationCommand( }) as ImageGenerationCommandV1 } -export function pendingImageGenerationCommands(workspace?: string): ImageGenerationCommand[] { - const pending: ImageGenerationCommand[] = [] - const area = storage() - for (let index = 0; index < area.length; index += 1) { - const key = area.key(index) - if (!key?.startsWith(PENDING_KEY_PREFIX)) continue - const intentId = key.slice(PENDING_KEY_PREFIX.length) - const command = readPending(intentId) - if (!command || (workspace !== undefined && command.input.workspace !== workspace)) continue - pending.push(command) - } - return pending.sort((left, right) => left.intent_id.localeCompare(right.intent_id)) -} - -export function pendingImageGenerationCommand( - intentId: string, - workspace?: string, -): ImageGenerationCommand | null { - const command = readPending(intentId) - if (!command || (workspace !== undefined && command.input.workspace !== workspace)) return null - return command -} - -function errorDetail(value: unknown): string | undefined { - if (typeof value === 'string' && value.trim()) return value - if (!isRecord(value)) return undefined - const detail = value.detail - if (typeof detail === 'string' && detail.trim()) return detail - if (isRecord(detail) && typeof detail.message === 'string' && detail.message.trim()) return detail.message - if (typeof value.message === 'string' && value.message.trim()) return value.message - return undefined -} +export const newImageGenerationIntentId = imageCommandClient.newIntentId +export const pendingImageGenerationCommands = imageCommandClient.pendingCommands +export const pendingImageGenerationCommand = imageCommandClient.pendingCommand -async function responseError( - response: Response, - intentId: string, - workspace: string, -): Promise { - const payload = await response.json().catch(() => undefined) - return new ImageGenerationCommandError( - errorDetail(payload) || `Image generation command failed (${response.status})`, - intentId, - workspace, - { status: response.status, uncertain: response.status >= 500, code: 'http_error' }, - ) -} - -interface ReceiptEnvelope { - receipt: unknown - replayed?: boolean - malformed?: boolean -} - -function unwrapReceipt(value: unknown): ReceiptEnvelope { - if (isRecord(value) && 'receipt' in value) { - if ('replayed' in value && typeof value.replayed !== 'boolean') { - return { receipt: value.receipt, malformed: true } - } - return { - receipt: value.receipt, - replayed: typeof value.replayed === 'boolean' ? value.replayed : undefined, - } - } - return { receipt: value } -} - -type ReceiptContext = Pick & { - input: Pick -} - -function invalidReceipt(command: ReceiptContext, message = 'Receipt could not be verified'): ImageGenerationCommandError { - return new ImageGenerationCommandError( - `${message} for ${command.intent_id}`, - command.intent_id, - command.input.workspace, - { status: 200, uncertain: true, code: 'invalid_receipt' }, - ) -} - -function validTaskResult(value: unknown, workspace: string, taskIds: unknown[]): value is ImageGenerationTaskResult { - return isRecord(value) - && typeof value.job_id === 'string' && value.job_id.length > 0 - && typeof value.task_id === 'string' && value.task_id.length > 0 - && taskIds.length === 1 && taskIds[0] === value.task_id - && value.workspace === workspace && value.status === 'queued' -} - -function receiptReplay(value: Record, command: ReceiptContext, fallback?: boolean): boolean | undefined { - if ('replayed' in value && typeof value.replayed !== 'boolean') throw invalidReceipt(command) - return typeof value.replayed === 'boolean' ? value.replayed : fallback -} - -function receiptV2Metadata(value: Record, command: ReceiptContext): { - commandVersion?: 2 - contentFingerprint?: string - fingerprintVersion?: 2 -} { - const metadataFields = ['commandVersion', 'contentFingerprint', 'fingerprintVersion'] as const - const present = metadataFields.filter(field => field in value) - // A receipt is either the complete v1 shape or the complete v2 shape. A - // half-present fingerprint must never be treated as a legacy receipt. - if (present.length !== 0 && present.length !== metadataFields.length) { - throw invalidReceipt(command, 'Receipt fingerprint metadata is incomplete') - } - if (present.length === 0) { - if (command.version === 2) { - throw invalidReceipt(command, 'The v2 receipt is missing its content fingerprint') - } - return {} - } - if (value.commandVersion !== 2 || value.fingerprintVersion !== 2 - || typeof value.contentFingerprint !== 'string' - || !/^[a-f0-9]{64}$/.test(value.contentFingerprint)) { - throw invalidReceipt(command, 'Receipt fingerprint metadata is invalid') - } - return { - commandVersion: 2, - contentFingerprint: value.contentFingerprint, - fingerprintVersion: 2, - } -} - -function validateReceipt( - value: unknown, - command: ReceiptContext, - replayed?: boolean, -): ImageGenerationReceipt { - if (!isRecord(value)) throw invalidReceipt(command) - const outerReplayed = receiptReplay(value, command, replayed) - const v2Metadata = receiptV2Metadata(value, command) - const result = value.result - const taskIds = value.taskIds - if (value.version !== IMAGE_GENERATION_SCHEMA_VERSION - || value.commandId !== command.intent_id - || value.operation !== command.operation - || value.status !== 'queued' - || !Array.isArray(value.entities) - || !Array.isArray(value.artifacts) - || !Array.isArray(taskIds) - || !Array.isArray(value.pipelineIds) - || !validTaskResult(result, command.input.workspace, taskIds)) { - throw invalidReceipt(command) - } - const receipt: ImageGenerationReceipt = { - version: IMAGE_GENERATION_SCHEMA_VERSION, - commandId: command.intent_id, - operation: IMAGE_GENERATION_OPERATION, - status: 'queued', - entities: JSON.parse(stableSerialize(value.entities)) as unknown[], - artifacts: JSON.parse(stableSerialize(value.artifacts)) as unknown[], - taskIds: [...taskIds] as string[], - pipelineIds: [...value.pipelineIds] as string[], - result: JSON.parse(stableSerialize(result)) as ImageGenerationTaskResult, - ...v2Metadata, - } - if (outerReplayed !== undefined) receipt.replayed = outerReplayed - return receipt -} - -function submissionContextPart(value: unknown, field: string): string | undefined { - if (value === undefined || value === null || value === '') return undefined - if (typeof value !== 'string' || !value.trim() || value.trim() !== value - || value.length > MAX_SUBMISSION_CONTEXT_ID_LENGTH) { - throw new Error(field + ' must be an exact non-blank string of at most 200 characters') - } - return value -} - -function validateSubmissionContext( - context: GenerationSubmissionContext | undefined, -): StoredSubmissionContext | undefined { - if (!context) return undefined - if (!SUBMISSION_ACTORS.has(context.actor)) { - throw new Error('submissionContext.actor must be a known actor') - } - const workflowId = submissionContextPart(context.workflowId, 'submissionContext.workflowId') - const runId = submissionContextPart(context.runId, 'submissionContext.runId') - return { - actor: context.actor, - ...(workflowId !== undefined ? { workflowId } : {}), - ...(runId !== undefined ? { runId } : {}), - } -} - -async function postCommand( - command: ImageGenerationCommand, - submissionContext?: StoredSubmissionContext, -): Promise { - let response: Response - try { - const headers: Record = { 'Content-Type': 'application/json' } - if (submissionContext) { - headers['X-Hocus-UI-Surface'] = submissionContext.actor === 'wizard' ? 'wizard' : 'studio' - const context: Record = {} - const workflowId = submissionContextPart(submissionContext.workflowId, 'submissionContext.workflowId') - const runId = submissionContextPart(submissionContext.runId, 'submissionContext.runId') - if (workflowId !== undefined) context.workflowId = workflowId - if (runId !== undefined) context.runId = runId - if (Object.keys(context).length > 0) headers['X-Hocus-UI-Context'] = JSON.stringify(context) - } - response = await fetch(`${BASE}/api/v1/generation/commands`, { - method: 'POST', - headers, - body: JSON.stringify(command), - }) - } catch (error) { - throw new ImageGenerationCommandError( - error instanceof Error ? error.message : 'Image generation request failed', - command.intent_id, - command.input.workspace, - { uncertain: true, code: 'transport_uncertain' }, - ) - } - if (!response.ok) { - throw await responseError(response, command.intent_id, command.input.workspace) - } - try { - return await response.json() - } catch { - throw new ImageGenerationCommandError( - `Image generation response could not be decoded for ${command.intent_id}`, - command.intent_id, - command.input.workspace, - { status: response.status, uncertain: true, code: 'invalid_response' }, - ) - } -} - -export interface SubmitImageGenerationCommandOptions { - /** - * A declared UI surface is transport metadata, never part of the command - * snapshot and never an authorization decision. - */ - submissionContext?: GenerationSubmissionContext - /** - * Runs after the pending hint is durable and before POST. The callback gets - * a detached copy so it cannot mutate the retry or transport snapshot. - */ - onSnapshotReady?: (snapshot: ImageGenerationCommand) => void | Promise -} - -interface PreparedSubmission { - recovering: boolean - submissionContext?: StoredSubmissionContext -} - -function requestedContext( - snapshot: ImageGenerationCommand, - context: GenerationSubmissionContext | undefined, -): StoredSubmissionContext | undefined { - try { - return validateSubmissionContext(context) - } catch (error) { - throw new ImageGenerationCommandError( - error instanceof Error ? error.message : 'The submission context is invalid', - snapshot.intent_id, - snapshot.input.workspace, - { code: 'invalid_submission_context' }, - ) - } -} - -function assertContextMatches( - snapshot: ImageGenerationCommand, - stored: StoredSubmissionContext | null, - requested: StoredSubmissionContext | undefined, -): void { - if (stored && requested && !sameSubmissionContext(stored, requested)) { - throw new ImageGenerationCommandError( - `intent_id ${snapshot.intent_id} is already attributed to a different UI context`, - snapshot.intent_id, - snapshot.input.workspace, - { code: 'submission_context_conflict', uncertain: true }, - ) - } -} - -function assertPendingCommandMatches( - snapshot: ImageGenerationCommand, - existing: ImageGenerationCommand | null, -): void { - if (existing && !sameCommand(existing, snapshot)) { - throw new ImageGenerationCommandError( - `intent_id ${snapshot.intent_id} is already pending with a different command`, - snapshot.intent_id, - snapshot.input.workspace, - { code: 'intent_conflict' }, - ) - } -} - -function prepareNewPendingContext( - snapshot: ImageGenerationCommand, - requested: StoredSubmissionContext | undefined, -): void { - // A sidecar is written before its command hint. If this write fails, no - // recoverable command is left behind to suggest an admitted request. - // Clearing an orphan is strict for the same reason: a stale attribution - // must not be paired with a newly written command. - persistPendingContext(snapshot, requested) -} - -function persistMissingRecoveryContext( - snapshot: ImageGenerationCommand, - recovering: boolean, - stored: StoredSubmissionContext | null, - requested: StoredSubmissionContext | undefined, -): StoredSubmissionContext | undefined { - const submissionContext = stored || requested - if (recovering && !stored && requested) { - persistPendingContext(snapshot, requested) - } - return submissionContext -} - -function preparePendingSubmission( - snapshot: ImageGenerationCommand, - requested: StoredSubmissionContext | undefined, -): PreparedSubmission { - let recovering = false - try { - const existing = readPending(snapshot.intent_id) - assertPendingCommandMatches(snapshot, existing) - recovering = existing !== null - if (!recovering) prepareNewPendingContext(snapshot, requested) - recovering = retainPending(snapshot) - const stored = recovering ? readPendingContext(snapshot) : null - assertContextMatches(snapshot, stored, requested) - return { - recovering, - submissionContext: persistMissingRecoveryContext(snapshot, recovering, stored, requested), - } - } catch (error) { - if (error instanceof ImageGenerationCommandError) throw error - throw new ImageGenerationCommandError( - error instanceof Error ? error.message : 'Could not persist image generation command', - snapshot.intent_id, - snapshot.input.workspace, - { code: 'pending_storage_failed', uncertain: recovering }, - ) - } -} - -async function presentSnapshot( - snapshot: ImageGenerationCommand, - recovering: boolean, - hook: SubmitImageGenerationCommandOptions['onSnapshotReady'], -): Promise { - if (!hook) return - try { - await hook(detachedCommand(snapshot)) - } catch (error) { - // A hook failure happens before network admission and is therefore - // certain. Preserve an older recovery hint because it may represent a - // previously admitted request whose response was lost. - if (!recovering) forgetPending(snapshot) - throw new ImageGenerationCommandError( - error instanceof Error ? error.message : 'The image command snapshot could not be presented', - snapshot.intent_id, - snapshot.input.workspace, - { code: 'snapshot_hook_failed' }, - ) - } -} - -async function admitImageCommand( - snapshot: ImageGenerationCommand, - submissionContext: StoredSubmissionContext | undefined, -): Promise { - const envelope = unwrapReceipt(await postCommand(snapshot, submissionContext)) - if (envelope.malformed) throw invalidReceipt(snapshot) - const receipt = validateReceipt(envelope.receipt, snapshot, envelope.replayed) - // A committed receipt is returned even if best-effort local cleanup fails. - forgetPending(snapshot) - return receipt -} - -function isDefinitiveClientError(error: ImageGenerationCommandError): boolean { - return error.status !== undefined && error.status >= 400 && error.status < 500 -} - -function normalizeSubmissionFailure( - error: unknown, - snapshot: ImageGenerationCommand, - recovering: boolean, -): ImageGenerationCommandError { - const commandError = error instanceof ImageGenerationCommandError - ? error - : new ImageGenerationCommandError( - error instanceof Error ? error.message : 'Image generation command failed', - snapshot.intent_id, - snapshot.input.workspace, - { uncertain: true, code: 'unknown_failure' }, - ) - // A first, explicit 4xx response is definitive before admission. Once a - // pending hint exists, a later rejection may follow an admitted request; - // preserve it, including a 401 after a timeout or lost response. - if (!recovering && isDefinitiveClientError(commandError)) forgetPending(snapshot) - return new ImageGenerationCommandError( - commandError.message, - snapshot.intent_id, - snapshot.input.workspace, - { - status: commandError.status, - uncertain: commandError.uncertain || recovering, - code: commandError.code, - }, - ) -} - -/** Submit or explicitly retry the same detached envelope and intention. */ -export async function submitImageGenerationCommand( - command: ImageGenerationCommand, - options: SubmitImageGenerationCommandOptions = {}, -): Promise { - const snapshot = detachedCommand(command) - const requestedSubmissionContext = requestedContext(snapshot, options.submissionContext) - const prepared = preparePendingSubmission(snapshot, requestedSubmissionContext) - - await presentSnapshot(snapshot, prepared.recovering, options.onSnapshotReady) - try { - return await admitImageCommand(snapshot, prepared.submissionContext) - } catch (error) { - throw normalizeSubmissionFailure(error, snapshot, prepared.recovering) - } -} - -function receiptCommandContext(intentId: string, workspace: string): ReceiptContext { - const fallback: ReceiptContext = { - version: 1, - intent_id: intentId, - operation: IMAGE_GENERATION_OPERATION, - input: { workspace }, - } - try { - const pending = readPending(intentId) - return pending && pending.input.workspace === workspace ? pending : fallback - } catch { - // A receipt read must remain available when local recovery storage is damaged. - return fallback - } -} - -async function requestReceipt(workspace: string, intentId: string): Promise { - try { - const response = await fetch( - `${BASE}/api/v1/generation/commands/receipt?workspace=${encodeURIComponent(workspace)}&intent_id=${encodeURIComponent(intentId)}`, - ) - if (!response.ok) throw await responseError(response, intentId, workspace) - return response - } catch (error) { - if (error instanceof ImageGenerationCommandError) throw error - throw new ImageGenerationCommandError( - error instanceof Error ? error.message : 'Receipt request failed', - intentId, - workspace, - { uncertain: true, code: 'transport_uncertain' }, - ) - } -} - -async function decodeReceiptResponse( - response: Response, - command: ReceiptContext, -): Promise { - try { - const payload = unwrapReceipt(await response.json()) - if (payload.malformed) throw invalidReceipt(command) - return validateReceipt(payload.receipt, command, payload.replayed) - } catch (error) { - if (error instanceof ImageGenerationCommandError) throw error - throw new ImageGenerationCommandError( - error instanceof Error ? error.message : `Receipt could not be verified for ${command.intent_id}`, - command.intent_id, - command.input.workspace, - { status: 200, uncertain: true, code: 'invalid_receipt' }, - ) - } -} - -function clearReceiptPending(intentId: string, workspace: string): void { - try { - const pending = readPending(intentId) - if (pending && pending.input.workspace === workspace) clearPendingReceipt(pending) - } catch { /* Keep the valid receipt even if local recovery storage is corrupt. */ } -} - -function clearPendingReceipt(command: ImageGenerationCommand): void { - try { retainPending(command, true) } catch { /* Keep the receipt visible if storage cleanup is unavailable. */ } -} - -/** Query a durable receipt after a lost response; this never invents a new ID. */ -export async function fetchImageGenerationCommandReceipt( - workspace: string, - intentId: string, -): Promise { - requiredText(workspace, 'workspace', MAX_ID_LENGTH) - requiredText(intentId, 'intent_id', MAX_INTENT_LENGTH) - // A receipt query has no command envelope of its own. When the durable hint - // is present, use its version so a lost v2 response cannot be accepted as a - // legacy v1 receipt. If the hint is unavailable, the endpoint remains a - // legacy-compatible read and the server's receipt metadata is still - // validated when it is present. - const receiptCommand = receiptCommandContext(intentId, workspace) - const response = await requestReceipt(workspace, intentId) - const receipt = await decodeReceiptResponse(response, receiptCommand) - // Receipt validation is authoritative. Storage read/removal is only a - // recovery hint and must never turn a valid GET into an apparent failure. - clearReceiptPending(intentId, workspace) - return receipt -} +export type SubmitImageGenerationCommandOptions = SubmitGenerationCommandOptions +export const submitImageGenerationCommand = imageCommandClient.submit +export const fetchImageGenerationCommandReceipt = imageCommandClient.fetchReceipt export const getImageGenerationCommandReceipt = fetchImageGenerationCommandReceipt export function subscribeImageGenerationCommands(callback: () => void): () => void { - window.addEventListener(PENDING_CHANGED_EVENT, callback) - window.addEventListener('storage', callback) - return () => { - window.removeEventListener(PENDING_CHANGED_EVENT, callback) - window.removeEventListener('storage', callback) - } + return imageCommandClient.subscribe(callback) } diff --git a/ui/src/api/musicCommandCatalog.json b/ui/src/api/musicCommandCatalog.json new file mode 100644 index 000000000..65f93d721 --- /dev/null +++ b/ui/src/api/musicCommandCatalog.json @@ -0,0 +1,1395 @@ +{ + "version": 2, + "operations": [ + { + "name": "generation.music", + "version": 2, + "supportedVersions": [ + 2 + ], + "domain": "studio", + "mutation": true, + "description": "Admit one local music generation with literal lyrics and caption through the canonical generation queue. The selected ACE-Step or MiniMax-Music3 model must already be installed; retries reuse the same intent receipt.", + "musicModelTypes": [ + "ace_step_v1_5_xl_sft_lm_4b", + "minimax_music3" + ], + "guideRevision": "music-model-contract-v1", + "inputSchema": { + "type": "object", + "additionalProperties": false, + "$defs": { + "StudioMusicCustomSettings": { + "additionalProperties": false, + "description": "Closed ACE-Step custom setting IDs shared by the native handler.", + "properties": { + "bpm": { + "anyOf": [ + { + "maximum": 300, + "minimum": 30, + "type": "integer" + }, + { + "const": "", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Bpm" + }, + "keyscale": { + "anyOf": [ + { + "maxLength": 8192, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Keyscale" + }, + "timesignature": { + "anyOf": [ + { + "enum": [ + 2, + 3, + 4, + 6, + "" + ] + }, + { + "type": "null" + } + ], + "default": null, + "title": "Timesignature" + }, + "language": { + "anyOf": [ + { + "maxLength": 8192, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Language" + } + }, + "title": "StudioMusicCustomSettings", + "type": "object" + }, + "StudioMusicParams": { + "additionalProperties": false, + "description": "Strict native music parameters emitted by Studio.", + "properties": { + "prompt": { + "maxLength": 200000, + "minLength": 1, + "title": "Prompt", + "type": "string" + }, + "alt_prompt": { + "default": "", + "maxLength": 200000, + "title": "Alt Prompt", + "type": "string" + }, + "model_type": { + "maxLength": 240, + "minLength": 1, + "title": "Model Type", + "type": "string" + }, + "resolution": { + "anyOf": [ + { + "maxLength": 8192, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Resolution" + }, + "lyrics_language": { + "anyOf": [ + { + "maxLength": 8192, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Lyrics Language" + }, + "video_length": { + "default": 0, + "maximum": 0, + "minimum": 0, + "title": "Video Length", + "type": "integer" + }, + "num_inference_steps": { + "anyOf": [ + { + "maximum": 1000, + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Num Inference Steps" + }, + "guidance_scale": { + "anyOf": [ + { + "maximum": 1000.0, + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Guidance Scale" + }, + "seed": { + "default": -1, + "maximum": 9223372036854775807, + "minimum": -9223372036854775808, + "title": "Seed", + "type": "integer" + }, + "image_mode": { + "default": 0, + "maximum": 0, + "minimum": 0, + "title": "Image Mode", + "type": "integer" + }, + "generation_mode": { + "const": "audio", + "default": "audio", + "title": "Generation Mode", + "type": "string" + }, + "negative_prompt": { + "default": "", + "enum": [ + "", + null + ], + "title": "Negative Prompt" + }, + "repeat_generation": { + "default": 1, + "maximum": 1, + "minimum": 1, + "title": "Repeat Generation", + "type": "integer" + }, + "batch_size": { + "default": 1, + "maximum": 1, + "minimum": 1, + "title": "Batch Size", + "type": "integer" + }, + "activated_loras": { + "items": { + "maxLength": 8192, + "minLength": 1, + "type": "string" + }, + "maxItems": 64, + "title": "Activated Loras", + "type": "array" + }, + "loras_multipliers": { + "default": "", + "maxLength": 8192, + "title": "Loras Multipliers", + "type": "string" + }, + "multi_prompts_gen_type": { + "default": 2, + "maximum": 2, + "minimum": 2, + "title": "Multi Prompts Gen Type", + "type": "integer" + }, + "audio_prompt_type": { + "default": "", + "enum": [ + "", + "A", + "B", + "AB" + ], + "title": "Audio Prompt Type", + "type": "string" + }, + "audio_guide": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide" + }, + "audio_guide2": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide2" + }, + "audio_guide3": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide3" + }, + "audio_guide4": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide4" + }, + "audio_guide5": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide5" + }, + "audio_guide6": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide6" + }, + "audio_source": { + "default": null, + "enum": [ + "", + null + ], + "title": "Audio Source" + }, + "duration_seconds": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Seconds" + }, + "temperature": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Temperature" + }, + "top_p": { + "anyOf": [ + { + "maximum": 1.0, + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Top P" + }, + "top_k": { + "anyOf": [ + { + "maximum": 1000, + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Top K" + }, + "audio_scale": { + "anyOf": [ + { + "maximum": 1000.0, + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Scale" + }, + "alt_guidance_scale": { + "anyOf": [ + { + "maximum": 1000.0, + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Alt Guidance Scale" + }, + "guidance_phases": { + "anyOf": [ + { + "maximum": 16, + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Guidance Phases" + }, + "sample_solver": { + "default": "", + "maxLength": 8192, + "title": "Sample Solver", + "type": "string" + }, + "settings_version": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Settings Version" + }, + "prompt_enhancer": { + "default": "", + "enum": [ + "", + null + ], + "title": "Prompt Enhancer" + }, + "model_mode": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Mode" + }, + "custom_settings": { + "anyOf": [ + { + "$ref": "#/$defs/StudioMusicCustomSettings" + }, + { + "type": "null" + } + ], + "default": null + }, + "_music_description": { + "default": "", + "maxLength": 200000, + "title": "Music Description", + "type": "string" + }, + "_music_instrumental": { + "default": false, + "title": "Music Instrumental", + "type": "boolean" + }, + "_audio_sub_mode": { + "const": "music", + "default": "music", + "title": "Audio Sub Mode", + "type": "string" + }, + "_tts_original_prompt": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Original Prompt" + }, + "_tts_speaker_name1": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name1" + }, + "_tts_speaker_name2": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name2" + }, + "_tts_speaker_name3": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name3" + }, + "_tts_speaker_name4": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name4" + }, + "_tts_speaker_name5": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name5" + }, + "_tts_speaker_name6": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name6" + }, + "_tts_voice_count": { + "default": 0, + "maximum": 0, + "minimum": 0, + "title": "Tts Voice Count", + "type": "integer" + } + }, + "required": [ + "prompt", + "model_type" + ], + "title": "StudioMusicParams", + "type": "object" + } + }, + "properties": { + "version": { + "type": "integer", + "const": 2 + }, + "operation": { + "const": "generation.music" + }, + "intent_id": { + "maxLength": 160, + "minLength": 1, + "title": "Intent Id", + "type": "string" + }, + "input": { + "additionalProperties": false, + "properties": { + "workspace": { + "maxLength": 240, + "minLength": 1, + "pattern": "^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$", + "title": "Workspace", + "type": "string" + }, + "workspace_collection_id": { + "anyOf": [ + { + "maxLength": 200, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Workspace Collection Id" + }, + "params": { + "$ref": "#/$defs/StudioMusicParams" + } + }, + "required": [ + "workspace", + "params" + ], + "title": "StudioMusicInput", + "type": "object" + } + }, + "required": [ + "version", + "operation", + "intent_id", + "input" + ] + } + } + ], + "studio": { + "version": 2, + "operation": "generation.music", + "intent_id": { + "maxLength": 160, + "minLength": 1, + "title": "Intent Id", + "type": "string" + }, + "input": { + "$defs": { + "StudioMusicCustomSettings": { + "additionalProperties": false, + "description": "Closed ACE-Step custom setting IDs shared by the native handler.", + "properties": { + "bpm": { + "anyOf": [ + { + "maximum": 300, + "minimum": 30, + "type": "integer" + }, + { + "const": "", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Bpm" + }, + "keyscale": { + "anyOf": [ + { + "maxLength": 8192, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Keyscale" + }, + "timesignature": { + "anyOf": [ + { + "enum": [ + 2, + 3, + 4, + 6, + "" + ] + }, + { + "type": "null" + } + ], + "default": null, + "title": "Timesignature" + }, + "language": { + "anyOf": [ + { + "maxLength": 8192, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Language" + } + }, + "title": "StudioMusicCustomSettings", + "type": "object" + }, + "StudioMusicParams": { + "additionalProperties": false, + "description": "Strict native music parameters emitted by Studio.", + "properties": { + "prompt": { + "maxLength": 200000, + "minLength": 1, + "title": "Prompt", + "type": "string" + }, + "alt_prompt": { + "default": "", + "maxLength": 200000, + "title": "Alt Prompt", + "type": "string" + }, + "model_type": { + "maxLength": 240, + "minLength": 1, + "title": "Model Type", + "type": "string" + }, + "resolution": { + "anyOf": [ + { + "maxLength": 8192, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Resolution" + }, + "lyrics_language": { + "anyOf": [ + { + "maxLength": 8192, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Lyrics Language" + }, + "video_length": { + "default": 0, + "maximum": 0, + "minimum": 0, + "title": "Video Length", + "type": "integer" + }, + "num_inference_steps": { + "anyOf": [ + { + "maximum": 1000, + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Num Inference Steps" + }, + "guidance_scale": { + "anyOf": [ + { + "maximum": 1000.0, + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Guidance Scale" + }, + "seed": { + "default": -1, + "maximum": 9223372036854775807, + "minimum": -9223372036854775808, + "title": "Seed", + "type": "integer" + }, + "image_mode": { + "default": 0, + "maximum": 0, + "minimum": 0, + "title": "Image Mode", + "type": "integer" + }, + "generation_mode": { + "const": "audio", + "default": "audio", + "title": "Generation Mode", + "type": "string" + }, + "negative_prompt": { + "default": "", + "enum": [ + "", + null + ], + "title": "Negative Prompt" + }, + "repeat_generation": { + "default": 1, + "maximum": 1, + "minimum": 1, + "title": "Repeat Generation", + "type": "integer" + }, + "batch_size": { + "default": 1, + "maximum": 1, + "minimum": 1, + "title": "Batch Size", + "type": "integer" + }, + "activated_loras": { + "items": { + "maxLength": 8192, + "minLength": 1, + "type": "string" + }, + "maxItems": 64, + "title": "Activated Loras", + "type": "array" + }, + "loras_multipliers": { + "default": "", + "maxLength": 8192, + "title": "Loras Multipliers", + "type": "string" + }, + "multi_prompts_gen_type": { + "default": 2, + "maximum": 2, + "minimum": 2, + "title": "Multi Prompts Gen Type", + "type": "integer" + }, + "audio_prompt_type": { + "default": "", + "enum": [ + "", + "A", + "B", + "AB" + ], + "title": "Audio Prompt Type", + "type": "string" + }, + "audio_guide": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide" + }, + "audio_guide2": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide2" + }, + "audio_guide3": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide3" + }, + "audio_guide4": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide4" + }, + "audio_guide5": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide5" + }, + "audio_guide6": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide6" + }, + "audio_source": { + "default": null, + "enum": [ + "", + null + ], + "title": "Audio Source" + }, + "duration_seconds": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Seconds" + }, + "temperature": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Temperature" + }, + "top_p": { + "anyOf": [ + { + "maximum": 1.0, + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Top P" + }, + "top_k": { + "anyOf": [ + { + "maximum": 1000, + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Top K" + }, + "audio_scale": { + "anyOf": [ + { + "maximum": 1000.0, + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Scale" + }, + "alt_guidance_scale": { + "anyOf": [ + { + "maximum": 1000.0, + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Alt Guidance Scale" + }, + "guidance_phases": { + "anyOf": [ + { + "maximum": 16, + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Guidance Phases" + }, + "sample_solver": { + "default": "", + "maxLength": 8192, + "title": "Sample Solver", + "type": "string" + }, + "settings_version": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Settings Version" + }, + "prompt_enhancer": { + "default": "", + "enum": [ + "", + null + ], + "title": "Prompt Enhancer" + }, + "model_mode": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Mode" + }, + "custom_settings": { + "anyOf": [ + { + "$ref": "#/$defs/StudioMusicCustomSettings" + }, + { + "type": "null" + } + ], + "default": null + }, + "_music_description": { + "default": "", + "maxLength": 200000, + "title": "Music Description", + "type": "string" + }, + "_music_instrumental": { + "default": false, + "title": "Music Instrumental", + "type": "boolean" + }, + "_audio_sub_mode": { + "const": "music", + "default": "music", + "title": "Audio Sub Mode", + "type": "string" + }, + "_tts_original_prompt": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Original Prompt" + }, + "_tts_speaker_name1": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name1" + }, + "_tts_speaker_name2": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name2" + }, + "_tts_speaker_name3": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name3" + }, + "_tts_speaker_name4": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name4" + }, + "_tts_speaker_name5": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name5" + }, + "_tts_speaker_name6": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name6" + }, + "_tts_voice_count": { + "default": 0, + "maximum": 0, + "minimum": 0, + "title": "Tts Voice Count", + "type": "integer" + } + }, + "required": [ + "prompt", + "model_type" + ], + "title": "StudioMusicParams", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "workspace": { + "maxLength": 240, + "minLength": 1, + "pattern": "^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$", + "title": "Workspace", + "type": "string" + }, + "workspace_collection_id": { + "anyOf": [ + { + "maxLength": 200, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Workspace Collection Id" + }, + "params": { + "$ref": "#/$defs/StudioMusicParams" + } + }, + "required": [ + "workspace", + "params" + ], + "title": "StudioMusicInput", + "type": "object" + }, + "supported_input_fields": [ + "prompt", + "alt_prompt", + "model_type", + "resolution", + "lyrics_language", + "video_length", + "num_inference_steps", + "guidance_scale", + "seed", + "image_mode", + "generation_mode", + "negative_prompt", + "repeat_generation", + "batch_size", + "activated_loras", + "loras_multipliers", + "audio_prompt_type", + "audio_guide", + "audio_guide2", + "audio_guide3", + "audio_guide4", + "audio_guide5", + "audio_guide6", + "audio_source", + "duration_seconds", + "temperature", + "top_p", + "top_k", + "audio_scale", + "alt_guidance_scale", + "guidance_phases", + "sample_solver", + "settings_version", + "prompt_enhancer", + "model_mode", + "custom_settings", + "multi_prompts_gen_type", + "_audio_sub_mode", + "_music_description", + "_music_instrumental", + "_tts_original_prompt", + "_tts_speaker_name1", + "_tts_speaker_name2", + "_tts_speaker_name3", + "_tts_speaker_name4", + "_tts_speaker_name5", + "_tts_speaker_name6", + "_tts_voice_count" + ], + "music_model_types": [ + "ace_step_v1_5_xl_sft_lm_4b", + "minimax_music3" + ], + "local_models": [ + "ace_step_v1_5_xl_sft_lm_4b", + "minimax_music3" + ], + "guide_revision": "music-model-contract-v1", + "effects": { + "generation_mode": "audio", + "_audio_sub_mode": "music", + "video_length": 0, + "image_mode": 0, + "multi_prompts_gen_type": 2, + "negative_prompt": "", + "repeat_generation": 1, + "batch_size": 1, + "activated_loras": [], + "loras_multipliers": "", + "audio_prompt_type": "", + "prompt_enhancer": "", + "_music_description": "", + "_music_instrumental": false, + "_tts_speaker_name1": "", + "_tts_speaker_name2": "", + "_tts_speaker_name3": "", + "_tts_speaker_name4": "", + "_tts_speaker_name5": "", + "_tts_speaker_name6": "", + "_tts_voice_count": 0 + }, + "inactive": [ + "image_mode=0", + "video_length=0", + "generation_mode=audio", + "_audio_sub_mode=music", + "multi_prompts_gen_type=2", + "repeat_generation=1", + "batch_size=1", + "negative_prompt=empty", + "prompt_enhancer=empty_or_null", + "_tts_speaker_name1..6=empty_or_null", + "_tts_voice_count=0", + "audio_guide3..6=empty_or_null", + "audio_source=empty_or_null" + ], + "excluded": [ + "actor", + "client", + "permission", + "provenance", + "workspace inside input.params", + "filesystem paths", + "free-form provider payloads", + "remote MiniMax and community model IDs", + "speech voice names, voice counts and voice-clone controls", + "SFX/MMAudio, video, image, avatar and model3d controls", + "LLM song-writing or prompt enhancement" + ] + } +} diff --git a/ui/src/api/musicGenerationCommands.ts b/ui/src/api/musicGenerationCommands.ts new file mode 100644 index 000000000..4f1b74444 --- /dev/null +++ b/ui/src/api/musicGenerationCommands.ts @@ -0,0 +1,89 @@ +import { + createGenerationCommandClient, + GenerationCommandError, + type GenerationReceiptLike, + type GenerationTaskResult, + type GenerationCommandErrorOptions, + type SubmitGenerationCommandOptions, +} from './generationCommandClient' +import { + assertStudioMusicGenerationCommand, + buildStudioMusicGenerationCommand, + createStudioMusicGenerationCommand, + detachedStudioMusicGenerationCommand, + type StudioMusicGenerationCommand, + type StudioMusicGenerationFullParams, + type StudioMusicGenerationInput, + type StudioMusicParamKey, + type StudioMusicParams, + STUDIO_MUSIC_OPERATION, + STUDIO_MUSIC_SCHEMA_VERSION, +} from '../features/studio/musicGenerationSpec' + +export { + assertStudioMusicGenerationCommand, + buildStudioMusicGenerationCommand, + createStudioMusicGenerationCommand, + detachedStudioMusicGenerationCommand, + STUDIO_MUSIC_OPERATION, + STUDIO_MUSIC_SCHEMA_VERSION, +} +export type { + StudioMusicGenerationCommand, + StudioMusicGenerationFullParams, + StudioMusicGenerationInput, + StudioMusicParamKey, + StudioMusicParams, +} + +export type MusicGenerationTaskResult = GenerationTaskResult + +export interface MusicGenerationReceipt extends GenerationReceiptLike { + version: 1 + operation: typeof STUDIO_MUSIC_OPERATION + result: MusicGenerationTaskResult +} + +export class MusicGenerationCommandError extends GenerationCommandError { + constructor( + message: string, + intentId: string, + workspace: string, + options: GenerationCommandErrorOptions = {}, + ) { + super(message, intentId, workspace, { + ...options, + code: options.code ?? 'music_generation_command_failed', + }) + this.name = 'MusicGenerationCommandError' + } +} + +const musicCommandClient = createGenerationCommandClient({ + storagePrefix: 'hocuspocus.generation.music-commands.v2:', + contextStoragePrefix: 'hocuspocus.generation.music-command-context.v1:', + pendingChangedEvent: 'hocuspocus:generation-music-commands-changed', + operation: STUDIO_MUSIC_OPERATION, + label: 'Music generation', + receiptFallbackVersion: STUDIO_MUSIC_SCHEMA_VERSION, + detach: detachedStudioMusicGenerationCommand, + errorClass: MusicGenerationCommandError, + castReceipt: value => value as MusicGenerationReceipt, +}) + +export const newMusicGenerationIntentId = musicCommandClient.newIntentId +export const pendingMusicGenerationCommands = musicCommandClient.pendingCommands +export const pendingMusicGenerationCommand = musicCommandClient.pendingCommand + +export type SubmitMusicGenerationCommandOptions = SubmitGenerationCommandOptions + +export const submitMusicGenerationCommand = musicCommandClient.submit +export const fetchMusicGenerationCommandReceipt = musicCommandClient.fetchReceipt +export const getMusicGenerationCommandReceipt = fetchMusicGenerationCommandReceipt + +export function subscribeMusicGenerationCommands(callback: () => void): () => void { + return musicCommandClient.subscribe(callback) +} + +/** Alias for callers which do not distinguish the Studio-specific name. */ +export const createMusicGenerationCommand = createStudioMusicGenerationCommand diff --git a/ui/src/api/sfxCommandCatalog.json b/ui/src/api/sfxCommandCatalog.json new file mode 100644 index 000000000..84b500d94 --- /dev/null +++ b/ui/src/api/sfxCommandCatalog.json @@ -0,0 +1,535 @@ +{ + "version": 2, + "operations": [ + { + "name": "generation.sfx", + "version": 2, + "supportedVersions": [ + 2 + ], + "domain": "studio", + "mutation": true, + "description": "Generate sound effects with installed MMAudio files in an explicit output workspace. Preserve literal prompts. Text-only requests produce audio with a duration up to 20 seconds; video-guided requests use the inspected video duration and produce a video with new audio. Sources require canonical references. Reuse intent_id only to recover an existing admission; follow its task for completion.", + "inputSchema": { + "type": "object", + "additionalProperties": false, + "$defs": { + "StudioSfxParams": { + "additionalProperties": false, + "description": "Typed native SFX parameters emitted by Studio.", + "properties": { + "prompt": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Prompt" + }, + "MMAudio_prompt": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Mmaudio Prompt" + }, + "MMAudio_neg_prompt": { + "default": "", + "maxLength": 200000, + "title": "Mmaudio Neg Prompt", + "type": "string" + }, + "model_type": { + "enum": [ + "mmaudio_v2", + "mmaudio_nsfw" + ], + "title": "Model Type", + "type": "string" + }, + "_mmaudio_variant": { + "anyOf": [ + { + "enum": [ + "v2", + "nsfw" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Mmaudio Variant" + }, + "duration_seconds": { + "exclusiveMinimum": 0.0, + "title": "Duration Seconds", + "type": "number" + }, + "seed": { + "default": -1, + "maximum": 9223372036854775807, + "minimum": -9223372036854775808, + "title": "Seed", + "type": "integer" + }, + "guidance_scale": { + "default": 4.5, + "maximum": 1000.0, + "minimum": 0.0, + "title": "Guidance Scale", + "type": "number" + }, + "sfx_text_weight": { + "default": 1.0, + "maximum": 5.0, + "minimum": 0.0, + "title": "Sfx Text Weight", + "type": "number" + }, + "num_inference_steps": { + "default": 25, + "maximum": 25, + "minimum": 25, + "title": "Num Inference Steps", + "type": "integer" + }, + "video_guide": { + "anyOf": [ + { + "maxLength": 8192, + "minLength": 1, + "type": "string" + }, + { + "enum": [ + "", + null + ] + } + ], + "default": null, + "title": "Video Guide" + }, + "generation_mode": { + "const": "audio", + "default": "audio", + "title": "Generation Mode", + "type": "string" + }, + "_audio_sub_mode": { + "const": "sfx", + "default": "sfx", + "title": "Audio Sub Mode", + "type": "string" + }, + "image_mode": { + "default": 0, + "maximum": 0, + "minimum": 0, + "title": "Image Mode", + "type": "integer" + }, + "video_length": { + "default": 0, + "maximum": 0, + "minimum": 0, + "title": "Video Length", + "type": "integer" + }, + "MMAudio_setting": { + "anyOf": [ + { + "const": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Mmaudio Setting" + }, + "sfx_mode": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sfx Mode" + } + }, + "required": [ + "model_type", + "duration_seconds" + ], + "title": "StudioSfxParams", + "type": "object" + } + }, + "properties": { + "version": { + "type": "integer", + "const": 2 + }, + "operation": { + "const": "generation.sfx" + }, + "intent_id": { + "maxLength": 160, + "minLength": 1, + "title": "Intent Id", + "type": "string" + }, + "input": { + "additionalProperties": false, + "properties": { + "workspace": { + "maxLength": 240, + "minLength": 1, + "pattern": "^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$", + "title": "Workspace", + "type": "string" + }, + "workspace_collection_id": { + "anyOf": [ + { + "maxLength": 200, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Workspace Collection Id" + }, + "params": { + "$ref": "#/$defs/StudioSfxParams" + } + }, + "required": [ + "workspace", + "params" + ], + "title": "StudioSfxInput", + "type": "object" + } + }, + "required": [ + "version", + "operation", + "intent_id", + "input" + ] + } + } + ], + "studio": { + "version": 2, + "operation": "generation.sfx", + "intent_id": { + "maxLength": 160, + "minLength": 1, + "title": "Intent Id", + "type": "string" + }, + "input": { + "$defs": { + "StudioSfxParams": { + "additionalProperties": false, + "description": "Typed native SFX parameters emitted by Studio.", + "properties": { + "prompt": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Prompt" + }, + "MMAudio_prompt": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Mmaudio Prompt" + }, + "MMAudio_neg_prompt": { + "default": "", + "maxLength": 200000, + "title": "Mmaudio Neg Prompt", + "type": "string" + }, + "model_type": { + "enum": [ + "mmaudio_v2", + "mmaudio_nsfw" + ], + "title": "Model Type", + "type": "string" + }, + "_mmaudio_variant": { + "anyOf": [ + { + "enum": [ + "v2", + "nsfw" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Mmaudio Variant" + }, + "duration_seconds": { + "exclusiveMinimum": 0.0, + "title": "Duration Seconds", + "type": "number" + }, + "seed": { + "default": -1, + "maximum": 9223372036854775807, + "minimum": -9223372036854775808, + "title": "Seed", + "type": "integer" + }, + "guidance_scale": { + "default": 4.5, + "maximum": 1000.0, + "minimum": 0.0, + "title": "Guidance Scale", + "type": "number" + }, + "sfx_text_weight": { + "default": 1.0, + "maximum": 5.0, + "minimum": 0.0, + "title": "Sfx Text Weight", + "type": "number" + }, + "num_inference_steps": { + "default": 25, + "maximum": 25, + "minimum": 25, + "title": "Num Inference Steps", + "type": "integer" + }, + "video_guide": { + "anyOf": [ + { + "maxLength": 8192, + "minLength": 1, + "type": "string" + }, + { + "enum": [ + "", + null + ] + } + ], + "default": null, + "title": "Video Guide" + }, + "generation_mode": { + "const": "audio", + "default": "audio", + "title": "Generation Mode", + "type": "string" + }, + "_audio_sub_mode": { + "const": "sfx", + "default": "sfx", + "title": "Audio Sub Mode", + "type": "string" + }, + "image_mode": { + "default": 0, + "maximum": 0, + "minimum": 0, + "title": "Image Mode", + "type": "integer" + }, + "video_length": { + "default": 0, + "maximum": 0, + "minimum": 0, + "title": "Video Length", + "type": "integer" + }, + "MMAudio_setting": { + "anyOf": [ + { + "const": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Mmaudio Setting" + }, + "sfx_mode": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sfx Mode" + } + }, + "required": [ + "model_type", + "duration_seconds" + ], + "title": "StudioSfxParams", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "workspace": { + "maxLength": 240, + "minLength": 1, + "pattern": "^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$", + "title": "Workspace", + "type": "string" + }, + "workspace_collection_id": { + "anyOf": [ + { + "maxLength": 200, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Workspace Collection Id" + }, + "params": { + "$ref": "#/$defs/StudioSfxParams" + } + }, + "required": [ + "workspace", + "params" + ], + "title": "StudioSfxInput", + "type": "object" + }, + "supported_input_fields": [ + "prompt", + "MMAudio_prompt", + "MMAudio_neg_prompt", + "model_type", + "_mmaudio_variant", + "duration_seconds", + "seed", + "guidance_scale", + "sfx_text_weight", + "video_guide", + "generation_mode", + "_audio_sub_mode", + "image_mode", + "video_length", + "num_inference_steps", + "MMAudio_setting", + "sfx_mode" + ], + "sfx_model_types": [ + "mmaudio_nsfw", + "mmaudio_v2" + ], + "variants": [ + "nsfw", + "v2" + ], + "effects": { + "generation_mode": "audio", + "_audio_sub_mode": "sfx", + "image_mode": 0, + "video_length": 0, + "num_inference_steps": 25, + "guidance_scale": 4.5, + "seed": -1, + "MMAudio_neg_prompt": "", + "sfx_text_weight": 1.0, + "MMAudio_setting": 1, + "sfx_mode": true + }, + "limits": { + "text_duration_seconds": { + "exclusive_minimum": 0, + "maximum": 20 + }, + "video_duration_seconds": "derived from inspected video_guide", + "video_requested_duration_seconds": { + "exclusive_minimum": 0 + } + }, + "inactive": [ + "generation_mode=audio", + "_audio_sub_mode=sfx", + "image_mode=0", + "video_length=0", + "audio_source=empty_or_null", + "video_source=empty_or_null", + "video_mask=empty_or_null", + "MMAudio_setting=1 (server-derived marker)", + "sfx_mode=true (server-derived marker)" + ], + "excluded": [ + "actor", + "client", + "permission", + "provenance", + "filesystem paths", + "remote URLs", + "free-form provider payloads", + "video carrier model selection", + "download controls", + "queue or recovery controls" + ] + } +} diff --git a/ui/src/api/sfxGenerationCommands.ts b/ui/src/api/sfxGenerationCommands.ts new file mode 100644 index 000000000..899eb5fd8 --- /dev/null +++ b/ui/src/api/sfxGenerationCommands.ts @@ -0,0 +1,83 @@ +import { + createGenerationCommandClient, + GenerationCommandError, + type GenerationReceiptLike, + type GenerationTaskResult, + type GenerationCommandErrorOptions, + type SubmitGenerationCommandOptions, +} from './generationCommandClient' +import { + assertStudioSfxGenerationCommand, + createStudioSfxGenerationCommand, + detachedStudioSfxGenerationCommand, + type StudioSfxGenerationCommand, + type StudioSfxParamKey, + type StudioSfxParams, + STUDIO_SFX_OPERATION, + STUDIO_SFX_SCHEMA_VERSION, +} from '../features/studio/sfxGenerationSpec' + +export { + assertStudioSfxGenerationCommand, + createStudioSfxGenerationCommand, + detachedStudioSfxGenerationCommand, + STUDIO_SFX_OPERATION, + STUDIO_SFX_SCHEMA_VERSION, +} +export type { + StudioSfxGenerationCommand, + StudioSfxParamKey, + StudioSfxParams, +} + +export type SfxGenerationTaskResult = GenerationTaskResult + +export interface SfxGenerationReceipt extends GenerationReceiptLike { + version: 1 + operation: typeof STUDIO_SFX_OPERATION + result: SfxGenerationTaskResult +} + +export class SfxGenerationCommandError extends GenerationCommandError { + constructor( + message: string, + intentId: string, + workspace: string, + options: GenerationCommandErrorOptions = {}, + ) { + super(message, intentId, workspace, { + ...options, + code: options.code ?? 'sfx_generation_command_failed', + }) + this.name = 'SfxGenerationCommandError' + } +} + +const sfxCommandClient = createGenerationCommandClient({ + storagePrefix: 'hocuspocus.generation.sfx-commands.v2:', + contextStoragePrefix: 'hocuspocus.generation.sfx-command-context.v1:', + pendingChangedEvent: 'hocuspocus:generation-sfx-commands-changed', + operation: STUDIO_SFX_OPERATION, + label: 'Sfx generation', + receiptFallbackVersion: STUDIO_SFX_SCHEMA_VERSION, + detach: detachedStudioSfxGenerationCommand, + errorClass: SfxGenerationCommandError, + castReceipt: value => value as SfxGenerationReceipt, +}) + +export const newSfxGenerationIntentId = sfxCommandClient.newIntentId +export const pendingSfxGenerationCommands = sfxCommandClient.pendingCommands +export const pendingSfxGenerationCommand = sfxCommandClient.pendingCommand + +export type SubmitSfxGenerationCommandOptions = SubmitGenerationCommandOptions + +export const submitSfxGenerationCommand = sfxCommandClient.submit +export const fetchSfxGenerationCommandReceipt = sfxCommandClient.fetchReceipt +export const getSfxGenerationCommandReceipt = fetchSfxGenerationCommandReceipt + +export function subscribeSfxGenerationCommands(callback: () => void): () => void { + return sfxCommandClient.subscribe(callback) +} + +/** Alias for callers which do not distinguish the Studio-specific name. */ +export const createSfxGenerationCommand = createStudioSfxGenerationCommand diff --git a/ui/src/api/speechCommandCatalog.json b/ui/src/api/speechCommandCatalog.json new file mode 100644 index 000000000..146d058fd --- /dev/null +++ b/ui/src/api/speechCommandCatalog.json @@ -0,0 +1,1973 @@ +{ + "version": 2, + "operations": [ + { + "name": "generation.speech", + "version": 2, + "supportedVersions": [ + 2 + ], + "domain": "studio", + "mutation": true, + "description": "Admit speech using an installed speech model, literal text, voice settings and canonical audio references in an explicit output workspace. Preserve the original speaker text separately from the effective native prompt. Reuse intent_id only for retries; the receipt proves admission, and its task reports completion.", + "inputSchema": { + "type": "object", + "additionalProperties": false, + "$defs": { + "StudioSpeechCustomSettings": { + "additionalProperties": false, + "description": "Typed union of custom settings exposed by speech handlers.\n\nThe selected model still owns which subset is valid and its metadata\nranges. Keeping the known IDs closed here prevents arbitrary nested JSON\nfrom crossing the command boundary while allowing each speech handler's\ncurrently published setting family.", + "properties": { + "auto_split_every_s": { + "anyOf": [ + { + "type": "number" + }, + { + "const": "", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Auto Split Every S" + }, + "exaggeration": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Exaggeration" + }, + "pace": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Pace" + }, + "vc_steps": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Vc Steps" + }, + "vc_cfg_rate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Vc Cfg Rate" + }, + "duration_multiplier": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Multiplier" + } + }, + "title": "StudioSpeechCustomSettings", + "type": "object" + }, + "StudioSpeechParams": { + "additionalProperties": false, + "description": "Typed native speech parameters emitted by Studio.", + "properties": { + "prompt": { + "maxLength": 200000, + "minLength": 1, + "title": "Prompt", + "type": "string" + }, + "alt_prompt": { + "default": "", + "maxLength": 200000, + "title": "Alt Prompt", + "type": "string" + }, + "model_type": { + "maxLength": 240, + "minLength": 1, + "title": "Model Type", + "type": "string" + }, + "resolution": { + "maxLength": 128, + "minLength": 1, + "title": "Resolution", + "type": "string" + }, + "video_length": { + "default": 0, + "maximum": 0, + "minimum": 0, + "title": "Video Length", + "type": "integer" + }, + "num_inference_steps": { + "default": 0, + "maximum": 100000, + "minimum": 0, + "title": "Num Inference Steps", + "type": "integer" + }, + "guidance_scale": { + "default": 1.0, + "maximum": 1000.0, + "minimum": 0.0, + "title": "Guidance Scale", + "type": "number" + }, + "seed": { + "default": -1, + "maximum": 9223372036854775807, + "minimum": -9223372036854775808, + "title": "Seed", + "type": "integer" + }, + "image_mode": { + "default": 0, + "maximum": 0, + "minimum": 0, + "title": "Image Mode", + "type": "integer" + }, + "generation_mode": { + "const": "audio", + "default": "audio", + "title": "Generation Mode", + "type": "string" + }, + "negative_prompt": { + "default": "", + "maxLength": 200000, + "title": "Negative Prompt", + "type": "string" + }, + "repeat_generation": { + "default": 1, + "maximum": 100, + "minimum": 1, + "title": "Repeat Generation", + "type": "integer" + }, + "activated_loras": { + "items": { + "maxLength": 8192, + "minLength": 1, + "type": "string" + }, + "maxItems": 64, + "title": "Activated Loras", + "type": "array" + }, + "loras_multipliers": { + "default": "", + "maxLength": 8192, + "title": "Loras Multipliers", + "type": "string" + }, + "multi_prompts_gen_type": { + "default": 2, + "maximum": 2, + "minimum": 2, + "title": "Multi Prompts Gen Type", + "type": "integer" + }, + "audio_prompt_type": { + "default": "", + "maxLength": 32, + "pattern": "^[A-Za-z0-9]*$", + "title": "Audio Prompt Type", + "type": "string" + }, + "audio_guide": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide" + }, + "audio_guide2": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide2" + }, + "audio_guide3": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide3" + }, + "audio_guide4": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide4" + }, + "audio_guide5": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide5" + }, + "audio_guide6": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide6" + }, + "audio_source": { + "default": null, + "enum": [ + "", + null + ], + "title": "Audio Source" + }, + "image_start": { + "anyOf": [ + { + "enum": [ + "", + null + ] + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], + "default": null, + "title": "Image Start" + }, + "image_end": { + "anyOf": [ + { + "enum": [ + "", + null + ] + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], + "default": null, + "title": "Image End" + }, + "image_refs": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Image Refs" + }, + "image_guide": { + "anyOf": [ + { + "enum": [ + "", + null + ] + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], + "default": null, + "title": "Image Guide" + }, + "image_mask": { + "anyOf": [ + { + "enum": [ + "", + null + ] + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], + "default": null, + "title": "Image Mask" + }, + "video_guide": { + "default": null, + "enum": [ + "", + null + ], + "title": "Video Guide" + }, + "video_mask": { + "default": null, + "enum": [ + "", + null + ], + "title": "Video Mask" + }, + "video_source": { + "default": null, + "enum": [ + "", + null + ], + "title": "Video Source" + }, + "spatial_upsampling": { + "default": "", + "enum": [ + "", + null + ], + "title": "Spatial Upsampling" + }, + "temporal_upsampling": { + "default": "", + "enum": [ + "", + null + ], + "title": "Temporal Upsampling" + }, + "wangp_processor_settings": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Wangp Processor Settings" + }, + "MMAudio_setting": { + "anyOf": [ + { + "maximum": 0, + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Mmaudio Setting" + }, + "MMAudio_prompt": { + "default": null, + "enum": [ + "", + null + ], + "title": "Mmaudio Prompt" + }, + "MMAudio_neg_prompt": { + "default": null, + "enum": [ + "", + null + ], + "title": "Mmaudio Neg Prompt" + }, + "h3_ref_videos": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "H3 Ref Videos" + }, + "h3_ref_audios": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "H3 Ref Audios" + }, + "minimax_h3_references": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Minimax H3 References" + }, + "_audio_sub_mode": { + "const": "speech", + "default": "speech", + "title": "Audio Sub Mode", + "type": "string" + }, + "_tts_original_prompt": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Original Prompt" + }, + "_tts_speaker_name1": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name1" + }, + "_tts_speaker_name2": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name2" + }, + "_tts_speaker_name3": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name3" + }, + "_tts_speaker_name4": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name4" + }, + "_tts_speaker_name5": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name5" + }, + "_tts_speaker_name6": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name6" + }, + "_tts_voice_count": { + "default": 0, + "maximum": 6, + "minimum": 0, + "title": "Tts Voice Count", + "type": "integer" + }, + "duration_seconds": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Seconds" + }, + "pause_seconds": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Pause Seconds" + }, + "temperature": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Temperature" + }, + "top_p": { + "anyOf": [ + { + "maximum": 1.0, + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Top P" + }, + "top_k": { + "anyOf": [ + { + "maximum": 100000, + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Top K" + }, + "audio_scale": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Scale" + }, + "audio_guidance_scale": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guidance Scale" + }, + "alt_scale": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Alt Scale" + }, + "guidance_phases": { + "default": 1, + "maximum": 16, + "minimum": 0, + "title": "Guidance Phases", + "type": "integer" + }, + "flow_shift": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Flow Shift" + }, + "sample_solver": { + "default": "", + "maxLength": 8192, + "title": "Sample Solver", + "type": "string" + }, + "settings_version": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Settings Version" + }, + "prompt_enhancer": { + "default": "", + "enum": [ + "", + null + ], + "title": "Prompt Enhancer" + }, + "model_mode": { + "anyOf": [ + { + "maxLength": 256, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Mode" + }, + "custom_settings": { + "anyOf": [ + { + "$ref": "#/$defs/StudioSpeechCustomSettings" + }, + { + "type": "null" + } + ], + "default": null + }, + "tts_dynaudnorm": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "integer" + }, + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Dynaudnorm" + }, + "tts_comp_threshold": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Comp Threshold" + }, + "tts_comp_attack": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Comp Attack" + }, + "tts_comp_release": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Comp Release" + }, + "tts_comp_makeup": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Comp Makeup" + }, + "minimax_h3_turbo_mode": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Minimax H3 Turbo Mode" + } + }, + "required": [ + "prompt", + "model_type", + "resolution" + ], + "title": "StudioSpeechParams", + "type": "object" + } + }, + "properties": { + "version": { + "type": "integer", + "const": 2 + }, + "operation": { + "const": "generation.speech" + }, + "intent_id": { + "maxLength": 160, + "minLength": 1, + "title": "Intent Id", + "type": "string" + }, + "input": { + "additionalProperties": false, + "properties": { + "workspace": { + "maxLength": 240, + "minLength": 1, + "pattern": "^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$", + "title": "Workspace", + "type": "string" + }, + "workspace_collection_id": { + "anyOf": [ + { + "maxLength": 200, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Workspace Collection Id" + }, + "params": { + "$ref": "#/$defs/StudioSpeechParams" + } + }, + "required": [ + "workspace", + "params" + ], + "title": "StudioSpeechInput", + "type": "object" + } + }, + "required": [ + "version", + "operation", + "intent_id", + "input" + ] + } + } + ], + "studio": { + "version": 2, + "operation": "generation.speech", + "intent_id": { + "maxLength": 160, + "minLength": 1, + "title": "Intent Id", + "type": "string" + }, + "input": { + "$defs": { + "StudioSpeechCustomSettings": { + "additionalProperties": false, + "description": "Typed union of custom settings exposed by speech handlers.\n\nThe selected model still owns which subset is valid and its metadata\nranges. Keeping the known IDs closed here prevents arbitrary nested JSON\nfrom crossing the command boundary while allowing each speech handler's\ncurrently published setting family.", + "properties": { + "auto_split_every_s": { + "anyOf": [ + { + "type": "number" + }, + { + "const": "", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Auto Split Every S" + }, + "exaggeration": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Exaggeration" + }, + "pace": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Pace" + }, + "vc_steps": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Vc Steps" + }, + "vc_cfg_rate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Vc Cfg Rate" + }, + "duration_multiplier": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Multiplier" + } + }, + "title": "StudioSpeechCustomSettings", + "type": "object" + }, + "StudioSpeechParams": { + "additionalProperties": false, + "description": "Typed native speech parameters emitted by Studio.", + "properties": { + "prompt": { + "maxLength": 200000, + "minLength": 1, + "title": "Prompt", + "type": "string" + }, + "alt_prompt": { + "default": "", + "maxLength": 200000, + "title": "Alt Prompt", + "type": "string" + }, + "model_type": { + "maxLength": 240, + "minLength": 1, + "title": "Model Type", + "type": "string" + }, + "resolution": { + "maxLength": 128, + "minLength": 1, + "title": "Resolution", + "type": "string" + }, + "video_length": { + "default": 0, + "maximum": 0, + "minimum": 0, + "title": "Video Length", + "type": "integer" + }, + "num_inference_steps": { + "default": 0, + "maximum": 100000, + "minimum": 0, + "title": "Num Inference Steps", + "type": "integer" + }, + "guidance_scale": { + "default": 1.0, + "maximum": 1000.0, + "minimum": 0.0, + "title": "Guidance Scale", + "type": "number" + }, + "seed": { + "default": -1, + "maximum": 9223372036854775807, + "minimum": -9223372036854775808, + "title": "Seed", + "type": "integer" + }, + "image_mode": { + "default": 0, + "maximum": 0, + "minimum": 0, + "title": "Image Mode", + "type": "integer" + }, + "generation_mode": { + "const": "audio", + "default": "audio", + "title": "Generation Mode", + "type": "string" + }, + "negative_prompt": { + "default": "", + "maxLength": 200000, + "title": "Negative Prompt", + "type": "string" + }, + "repeat_generation": { + "default": 1, + "maximum": 100, + "minimum": 1, + "title": "Repeat Generation", + "type": "integer" + }, + "activated_loras": { + "items": { + "maxLength": 8192, + "minLength": 1, + "type": "string" + }, + "maxItems": 64, + "title": "Activated Loras", + "type": "array" + }, + "loras_multipliers": { + "default": "", + "maxLength": 8192, + "title": "Loras Multipliers", + "type": "string" + }, + "multi_prompts_gen_type": { + "default": 2, + "maximum": 2, + "minimum": 2, + "title": "Multi Prompts Gen Type", + "type": "integer" + }, + "audio_prompt_type": { + "default": "", + "maxLength": 32, + "pattern": "^[A-Za-z0-9]*$", + "title": "Audio Prompt Type", + "type": "string" + }, + "audio_guide": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide" + }, + "audio_guide2": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide2" + }, + "audio_guide3": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide3" + }, + "audio_guide4": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide4" + }, + "audio_guide5": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide5" + }, + "audio_guide6": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide6" + }, + "audio_source": { + "default": null, + "enum": [ + "", + null + ], + "title": "Audio Source" + }, + "image_start": { + "anyOf": [ + { + "enum": [ + "", + null + ] + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], + "default": null, + "title": "Image Start" + }, + "image_end": { + "anyOf": [ + { + "enum": [ + "", + null + ] + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], + "default": null, + "title": "Image End" + }, + "image_refs": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Image Refs" + }, + "image_guide": { + "anyOf": [ + { + "enum": [ + "", + null + ] + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], + "default": null, + "title": "Image Guide" + }, + "image_mask": { + "anyOf": [ + { + "enum": [ + "", + null + ] + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], + "default": null, + "title": "Image Mask" + }, + "video_guide": { + "default": null, + "enum": [ + "", + null + ], + "title": "Video Guide" + }, + "video_mask": { + "default": null, + "enum": [ + "", + null + ], + "title": "Video Mask" + }, + "video_source": { + "default": null, + "enum": [ + "", + null + ], + "title": "Video Source" + }, + "spatial_upsampling": { + "default": "", + "enum": [ + "", + null + ], + "title": "Spatial Upsampling" + }, + "temporal_upsampling": { + "default": "", + "enum": [ + "", + null + ], + "title": "Temporal Upsampling" + }, + "wangp_processor_settings": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Wangp Processor Settings" + }, + "MMAudio_setting": { + "anyOf": [ + { + "maximum": 0, + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Mmaudio Setting" + }, + "MMAudio_prompt": { + "default": null, + "enum": [ + "", + null + ], + "title": "Mmaudio Prompt" + }, + "MMAudio_neg_prompt": { + "default": null, + "enum": [ + "", + null + ], + "title": "Mmaudio Neg Prompt" + }, + "h3_ref_videos": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "H3 Ref Videos" + }, + "h3_ref_audios": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "H3 Ref Audios" + }, + "minimax_h3_references": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Minimax H3 References" + }, + "_audio_sub_mode": { + "const": "speech", + "default": "speech", + "title": "Audio Sub Mode", + "type": "string" + }, + "_tts_original_prompt": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Original Prompt" + }, + "_tts_speaker_name1": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name1" + }, + "_tts_speaker_name2": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name2" + }, + "_tts_speaker_name3": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name3" + }, + "_tts_speaker_name4": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name4" + }, + "_tts_speaker_name5": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name5" + }, + "_tts_speaker_name6": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name6" + }, + "_tts_voice_count": { + "default": 0, + "maximum": 6, + "minimum": 0, + "title": "Tts Voice Count", + "type": "integer" + }, + "duration_seconds": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Seconds" + }, + "pause_seconds": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Pause Seconds" + }, + "temperature": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Temperature" + }, + "top_p": { + "anyOf": [ + { + "maximum": 1.0, + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Top P" + }, + "top_k": { + "anyOf": [ + { + "maximum": 100000, + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Top K" + }, + "audio_scale": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Scale" + }, + "audio_guidance_scale": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guidance Scale" + }, + "alt_scale": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Alt Scale" + }, + "guidance_phases": { + "default": 1, + "maximum": 16, + "minimum": 0, + "title": "Guidance Phases", + "type": "integer" + }, + "flow_shift": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Flow Shift" + }, + "sample_solver": { + "default": "", + "maxLength": 8192, + "title": "Sample Solver", + "type": "string" + }, + "settings_version": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Settings Version" + }, + "prompt_enhancer": { + "default": "", + "enum": [ + "", + null + ], + "title": "Prompt Enhancer" + }, + "model_mode": { + "anyOf": [ + { + "maxLength": 256, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Mode" + }, + "custom_settings": { + "anyOf": [ + { + "$ref": "#/$defs/StudioSpeechCustomSettings" + }, + { + "type": "null" + } + ], + "default": null + }, + "tts_dynaudnorm": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "integer" + }, + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Dynaudnorm" + }, + "tts_comp_threshold": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Comp Threshold" + }, + "tts_comp_attack": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Comp Attack" + }, + "tts_comp_release": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Comp Release" + }, + "tts_comp_makeup": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Comp Makeup" + }, + "minimax_h3_turbo_mode": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Minimax H3 Turbo Mode" + } + }, + "required": [ + "prompt", + "model_type", + "resolution" + ], + "title": "StudioSpeechParams", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "workspace": { + "maxLength": 240, + "minLength": 1, + "pattern": "^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$", + "title": "Workspace", + "type": "string" + }, + "workspace_collection_id": { + "anyOf": [ + { + "maxLength": 200, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Workspace Collection Id" + }, + "params": { + "$ref": "#/$defs/StudioSpeechParams" + } + }, + "required": [ + "workspace", + "params" + ], + "title": "StudioSpeechInput", + "type": "object" + }, + "supported_input_fields": [ + "prompt", + "alt_prompt", + "model_type", + "resolution", + "video_length", + "num_inference_steps", + "guidance_scale", + "seed", + "image_mode", + "generation_mode", + "negative_prompt", + "repeat_generation", + "activated_loras", + "loras_multipliers", + "audio_prompt_type", + "audio_guide", + "audio_guide2", + "audio_guide3", + "audio_guide4", + "audio_guide5", + "audio_guide6", + "audio_source", + "_audio_sub_mode", + "_tts_original_prompt", + "_tts_speaker_name1", + "_tts_speaker_name2", + "_tts_speaker_name3", + "_tts_speaker_name4", + "_tts_speaker_name5", + "_tts_speaker_name6", + "_tts_voice_count", + "duration_seconds", + "pause_seconds", + "temperature", + "top_p", + "top_k", + "audio_scale", + "audio_guidance_scale", + "alt_scale", + "guidance_phases", + "flow_shift", + "sample_solver", + "settings_version", + "prompt_enhancer", + "model_mode", + "custom_settings", + "tts_dynaudnorm", + "tts_comp_threshold", + "tts_comp_attack", + "tts_comp_release", + "tts_comp_makeup", + "multi_prompts_gen_type", + "minimax_h3_turbo_mode", + "image_start", + "image_end", + "image_refs", + "image_guide", + "image_mask", + "video_guide", + "video_mask", + "video_source", + "spatial_upsampling", + "temporal_upsampling", + "wangp_processor_settings", + "MMAudio_setting", + "MMAudio_prompt", + "MMAudio_neg_prompt", + "h3_ref_videos", + "h3_ref_audios", + "minimax_h3_references" + ], + "speech_model_types": [ + "chatterbox", + "dramabox_audio", + "index_tts2", + "kugelaudio_0_open", + "qwen3_tts_base", + "qwen3_tts_customvoice", + "qwen3_tts_voicedesign", + "scenema_audio" + ], + "effects": { + "generation_mode": "audio", + "_audio_sub_mode": "speech", + "video_length": 0, + "image_mode": 0, + "multi_prompts_gen_type": 2, + "negative_prompt": "", + "repeat_generation": 1, + "activated_loras": [], + "loras_multipliers": "", + "audio_prompt_type": "", + "prompt_enhancer": "", + "minimax_h3_turbo_mode": false, + "_tts_speaker_name1": "", + "_tts_speaker_name2": "", + "_tts_speaker_name3": "", + "_tts_speaker_name4": "", + "_tts_speaker_name5": "", + "_tts_speaker_name6": "", + "_tts_voice_count": 0 + }, + "inactive": [ + "image_mode", + "video_length", + "minimax_h3_turbo_mode", + "image_start", + "image_end", + "image_refs", + "image_guide", + "image_mask", + "video_guide", + "video_mask", + "video_source", + "audio_source", + "MMAudio_setting", + "MMAudio_prompt", + "MMAudio_neg_prompt", + "h3_ref_videos", + "h3_ref_audios", + "minimax_h3_references", + "spatial_upsampling", + "temporal_upsampling", + "wangp_processor_settings" + ], + "excluded": [ + "actor", + "client", + "permission", + "provenance", + "workspace inside input.params", + "filesystem paths", + "free-form provider payloads", + "music and SFX model types", + "video, avatar and model3d controls" + ] + } +} diff --git a/ui/src/api/speechGenerationCommands.ts b/ui/src/api/speechGenerationCommands.ts new file mode 100644 index 000000000..d17dc8490 --- /dev/null +++ b/ui/src/api/speechGenerationCommands.ts @@ -0,0 +1,92 @@ +import { + createGenerationCommandClient, + GenerationCommandError, + type GenerationReceiptLike, + type GenerationTaskResult, + type GenerationCommandErrorOptions, + type SubmitGenerationCommandOptions, +} from './generationCommandClient' +import { + assertStudioSpeechGenerationCommand, + buildStudioSpeechGenerationCommand, + createStudioSpeechGenerationCommand, + detachedStudioSpeechGenerationCommand, + type StudioSpeechGenerationCommand, + type StudioSpeechGenerationFullParams, + type StudioSpeechGenerationInput, + type StudioSpeechParamKey, + type StudioSpeechParams, + STUDIO_SPEECH_OPERATION, + STUDIO_SPEECH_SCHEMA_VERSION, +} from '../features/studio/speechGenerationSpec' + +export { + assertStudioSpeechGenerationCommand, + buildStudioSpeechGenerationCommand, + createStudioSpeechGenerationCommand, + detachedStudioSpeechGenerationCommand, + STUDIO_SPEECH_OPERATION, + STUDIO_SPEECH_SCHEMA_VERSION, +} +export type { + StudioSpeechGenerationCommand, + StudioSpeechGenerationFullParams, + StudioSpeechGenerationInput, + StudioSpeechParamKey, + StudioSpeechParams, +} + +export type SpeechGenerationTaskResult = GenerationTaskResult + +export interface SpeechGenerationReceipt extends GenerationReceiptLike { + version: 1 + operation: typeof STUDIO_SPEECH_OPERATION + result: SpeechGenerationTaskResult +} + +export class SpeechGenerationCommandError extends GenerationCommandError { + constructor( + message: string, + intentId: string, + workspace: string, + options: GenerationCommandErrorOptions = {}, + ) { + super(message, intentId, workspace, { + ...options, + code: options.code ?? 'speech_generation_command_failed', + }) + this.name = 'SpeechGenerationCommandError' + } +} + +const speechCommandClient = createGenerationCommandClient({ + storagePrefix: 'hocuspocus.generation.speech-commands.v2:', + contextStoragePrefix: 'hocuspocus.generation.speech-command-context.v1:', + pendingChangedEvent: 'hocuspocus:generation-speech-commands-changed', + operation: STUDIO_SPEECH_OPERATION, + label: 'Speech generation', + receiptFallbackVersion: STUDIO_SPEECH_SCHEMA_VERSION, + detach: detachedStudioSpeechGenerationCommand, + errorClass: SpeechGenerationCommandError, + castReceipt: value => value as SpeechGenerationReceipt, +}) + +export const newSpeechGenerationIntentId = speechCommandClient.newIntentId +export const pendingSpeechGenerationCommands = speechCommandClient.pendingCommands +export const pendingSpeechGenerationCommand = speechCommandClient.pendingCommand + +export type SubmitSpeechGenerationCommandOptions = SubmitGenerationCommandOptions + +export const submitSpeechGenerationCommand = speechCommandClient.submit +export const fetchSpeechGenerationCommandReceipt = speechCommandClient.fetchReceipt +export const getSpeechGenerationCommandReceipt = fetchSpeechGenerationCommandReceipt + +export function subscribeSpeechGenerationCommands(callback: () => void): () => void { + return speechCommandClient.subscribe(callback) +} + +/** + * Symmetric name for callers that already distinguish legacy Speech from the + * typed Studio envelope. The builder still receives the complete native map. + */ +export const createSpeechGenerationCommand = createStudioSpeechGenerationCommand diff --git a/ui/src/api/toolsCommandCatalog.json b/ui/src/api/toolsCommandCatalog.json new file mode 100644 index 000000000..20d2e07ff --- /dev/null +++ b/ui/src/api/toolsCommandCatalog.json @@ -0,0 +1,491 @@ +{ + "version": 2, + "operations": [ + { + "name": "tools.upscale", + "version": 2, + "supportedVersions": [ + 2 + ], + "domain": "tools", + "mutation": true, + "description": "Upscale one exact image or video source in an explicit output workspace with an installed local processor. Preserve the source workspace, processor settings and intent_id; the shared receipt proves admission and its canonical task reports completion.", + "inputSchema": { + "type": "object", + "additionalProperties": false, + "$defs": { + "ToolsUpscaleParams": { + "additionalProperties": false, + "description": "Typed native input for one image or video upscale.", + "properties": { + "source": { + "maxLength": 8192, + "minLength": 1, + "title": "Source", + "type": "string" + }, + "source_workspace": { + "anyOf": [ + { + "maxLength": 160, + "minLength": 1, + "pattern": "^(?:__uploads__|default|[A-Za-z0-9][A-Za-z0-9_-]*)$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Workspace" + }, + "source_kind": { + "enum": [ + "image", + "video" + ], + "title": "Source Kind", + "type": "string" + }, + "method": { + "maxLength": 80, + "minLength": 1, + "title": "Method", + "type": "string" + }, + "seed": { + "default": -1, + "maximum": 9223372036854775807, + "minimum": -9223372036854775808, + "title": "Seed", + "type": "integer" + }, + "wangp_processor_settings": { + "anyOf": [ + { + "$ref": "#/$defs/WangpProcessorSettings" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "required": [ + "source", + "source_kind", + "method" + ], + "title": "ToolsUpscaleParams", + "type": "object" + }, + "WangpProcessorSettings": { + "additionalProperties": false, + "description": "Typed settings currently declared by image-capable processors.", + "properties": { + "spatial_upsampler_strength": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Spatial Upsampler Strength" + }, + "spatial_upsampler_face_count": { + "anyOf": [ + { + "maximum": 5, + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Spatial Upsampler Face Count" + }, + "spatial_upsampler_h3_strength": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Spatial Upsampler H3 Strength" + }, + "spatial_upsampler_prompt": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Spatial Upsampler Prompt" + }, + "spatial_upsampler_reference_images": { + "anyOf": [ + { + "items": { + "maxLength": 8192, + "minLength": 1, + "type": "string" + }, + "maxItems": 64, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Spatial Upsampler Reference Images" + }, + "spatial_upsampler_dlss_strength": { + "anyOf": [ + { + "maximum": 2, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Spatial Upsampler Dlss Strength" + } + }, + "title": "WangpProcessorSettings", + "type": "object" + } + }, + "properties": { + "version": { + "type": "integer", + "const": 2 + }, + "operation": { + "const": "tools.upscale" + }, + "intent_id": { + "type": "string", + "minLength": 1, + "maxLength": 160 + }, + "input": { + "additionalProperties": false, + "properties": { + "workspace": { + "maxLength": 240, + "minLength": 1, + "pattern": "^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$", + "title": "Workspace", + "type": "string" + }, + "workspace_collection_id": { + "anyOf": [ + { + "maxLength": 200, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Workspace Collection Id" + }, + "params": { + "$ref": "#/$defs/ToolsUpscaleParams" + } + }, + "required": [ + "workspace", + "params" + ], + "title": "ToolsUpscaleInput", + "type": "object" + } + }, + "required": [ + "version", + "operation", + "intent_id", + "input" + ] + } + } + ], + "studio": { + "version": 2, + "operation": "tools.upscale", + "intent_id": { + "type": "string", + "minLength": 1, + "maxLength": 160 + }, + "input": { + "$defs": { + "ToolsUpscaleParams": { + "additionalProperties": false, + "description": "Typed native input for one image or video upscale.", + "properties": { + "source": { + "maxLength": 8192, + "minLength": 1, + "title": "Source", + "type": "string" + }, + "source_workspace": { + "anyOf": [ + { + "maxLength": 160, + "minLength": 1, + "pattern": "^(?:__uploads__|default|[A-Za-z0-9][A-Za-z0-9_-]*)$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Workspace" + }, + "source_kind": { + "enum": [ + "image", + "video" + ], + "title": "Source Kind", + "type": "string" + }, + "method": { + "maxLength": 80, + "minLength": 1, + "title": "Method", + "type": "string" + }, + "seed": { + "default": -1, + "maximum": 9223372036854775807, + "minimum": -9223372036854775808, + "title": "Seed", + "type": "integer" + }, + "wangp_processor_settings": { + "anyOf": [ + { + "$ref": "#/$defs/WangpProcessorSettings" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "required": [ + "source", + "source_kind", + "method" + ], + "title": "ToolsUpscaleParams", + "type": "object" + }, + "WangpProcessorSettings": { + "additionalProperties": false, + "description": "Typed settings currently declared by image-capable processors.", + "properties": { + "spatial_upsampler_strength": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Spatial Upsampler Strength" + }, + "spatial_upsampler_face_count": { + "anyOf": [ + { + "maximum": 5, + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Spatial Upsampler Face Count" + }, + "spatial_upsampler_h3_strength": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Spatial Upsampler H3 Strength" + }, + "spatial_upsampler_prompt": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Spatial Upsampler Prompt" + }, + "spatial_upsampler_reference_images": { + "anyOf": [ + { + "items": { + "maxLength": 8192, + "minLength": 1, + "type": "string" + }, + "maxItems": 64, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Spatial Upsampler Reference Images" + }, + "spatial_upsampler_dlss_strength": { + "anyOf": [ + { + "maximum": 2, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Spatial Upsampler Dlss Strength" + } + }, + "title": "WangpProcessorSettings", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "workspace": { + "maxLength": 240, + "minLength": 1, + "pattern": "^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$", + "title": "Workspace", + "type": "string" + }, + "workspace_collection_id": { + "anyOf": [ + { + "maxLength": 200, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Workspace Collection Id" + }, + "params": { + "$ref": "#/$defs/ToolsUpscaleParams" + } + }, + "required": [ + "workspace", + "params" + ], + "title": "ToolsUpscaleInput", + "type": "object" + }, + "supported_input_fields": [ + "workspace", + "workspace_collection_id", + "source", + "source_workspace", + "source_kind", + "method", + "seed", + "wangp_processor_settings" + ], + "source_kinds": [ + "image", + "video" + ], + "methods": [ + "dlss5*1", + "dlss5*1.5", + "dlss5*1.724", + "dlss5*2", + "dlss5*3", + "dlssg*2", + "dlssg*3", + "dlssg*4", + "dlssg*5", + "dlssg*6", + "flashvsr2", + "flashvsr2pass2", + "flashvsr2pass4", + "flashvsr3", + "flashvsr4", + "h3facerefine", + "lanczos1.5", + "lanczos2", + "rife2", + "rife3", + "rife4" + ], + "effects": { + "seed": -1, + "wangp_processor_settings": {} + }, + "excluded": [ + "actor", + "client", + "permission", + "provenance", + "workspace inside input.params", + "filesystem paths", + "remote URLs", + "free-form processor settings", + "generation.image", + "generation.speech" + ] + } +} diff --git a/ui/src/api/toolsGenerationCommands.ts b/ui/src/api/toolsGenerationCommands.ts new file mode 100644 index 000000000..5e74f4b65 --- /dev/null +++ b/ui/src/api/toolsGenerationCommands.ts @@ -0,0 +1,89 @@ +import { + createGenerationCommandClient, + GenerationCommandError, + type GenerationCommandErrorOptions, + type GenerationReceiptLike, + type GenerationTaskResult, + type SubmitGenerationCommandOptions, +} from './generationCommandClient' +import { + buildStudioToolsUpscaleGenerationCommand, + createStudioToolsUpscaleGenerationCommand, + detachedToolsUpscaleGenerationCommand, + TOOLS_UPSCALE_OPERATION, + TOOLS_UPSCALE_SCHEMA_VERSION, + type ToolsUpscaleGenerationCommand, +} from '../features/studio/toolsGenerationSpec' + +export { + assertToolsUpscaleGenerationCommand, + buildStudioToolsUpscaleGenerationCommand, + canonicalToolsSource, + createStudioToolsUpscaleGenerationCommand, + detachedToolsUpscaleGenerationCommand, + TOOLS_UPSCALE_OPERATION, + TOOLS_UPSCALE_SCHEMA_VERSION, +} from '../features/studio/toolsGenerationSpec' +export type { + ToolsUpscaleGenerationCommand, + ToolsUpscaleGenerationFullParams, + ToolsUpscaleGenerationInput, + ToolsUpscaleParamKey, + ToolsUpscaleParams, +} from '../features/studio/toolsGenerationSpec' + +export type ToolsUpscaleTaskResult = GenerationTaskResult + +export interface ToolsUpscaleGenerationReceipt extends GenerationReceiptLike { + version: 1 + operation: typeof TOOLS_UPSCALE_OPERATION + result: ToolsUpscaleTaskResult +} + +export class ToolsUpscaleGenerationCommandError extends GenerationCommandError { + constructor( + message: string, + intentId: string, + workspace: string, + options: GenerationCommandErrorOptions = {}, + ) { + super(message, intentId, workspace, { + ...options, + code: options.code ?? 'tools_upscale_command_failed', + }) + this.name = 'ToolsUpscaleGenerationCommandError' + } +} + +const toolsUpscaleCommandClient = createGenerationCommandClient< + ToolsUpscaleGenerationCommand, + ToolsUpscaleGenerationReceipt +>({ + storagePrefix: 'hocuspocus.generation.tools-upscale-commands.v2:', + contextStoragePrefix: 'hocuspocus.generation.tools-upscale-command-context.v1:', + pendingChangedEvent: 'hocuspocus:generation-tools-upscale-commands-changed', + operation: TOOLS_UPSCALE_OPERATION, + label: 'Tools upscale', + receiptFallbackVersion: TOOLS_UPSCALE_SCHEMA_VERSION, + detach: detachedToolsUpscaleGenerationCommand, + errorClass: ToolsUpscaleGenerationCommandError, + castReceipt: value => value as ToolsUpscaleGenerationReceipt, +}) + +export const newToolsUpscaleGenerationIntentId = toolsUpscaleCommandClient.newIntentId +export const pendingToolsUpscaleGenerationCommands = toolsUpscaleCommandClient.pendingCommands +export const pendingToolsUpscaleGenerationCommand = toolsUpscaleCommandClient.pendingCommand + +export type SubmitToolsUpscaleGenerationCommandOptions = SubmitGenerationCommandOptions + +export const submitToolsUpscaleGenerationCommand = toolsUpscaleCommandClient.submit +export const fetchToolsUpscaleGenerationCommandReceipt = toolsUpscaleCommandClient.fetchReceipt +export const getToolsUpscaleGenerationCommandReceipt = fetchToolsUpscaleGenerationCommandReceipt + +export function subscribeToolsUpscaleGenerationCommands(callback: () => void): () => void { + return toolsUpscaleCommandClient.subscribe(callback) +} + +/** Symmetric short aliases used by Tools callers. */ +export const createToolsUpscaleCommand = createStudioToolsUpscaleGenerationCommand +export const buildToolsUpscaleCommand = buildStudioToolsUpscaleGenerationCommand diff --git a/ui/src/components/MainContent/MediaFeedItem.tsx b/ui/src/components/MainContent/MediaFeedItem.tsx index 82586d9ce..3d9eb01fe 100644 --- a/ui/src/components/MainContent/MediaFeedItem.tsx +++ b/ui/src/components/MainContent/MediaFeedItem.tsx @@ -135,6 +135,9 @@ export function MediaFeedItem({ file, index, isActive, onVisible, onMeasured, ma const [selectingForMontage, setSelectingForMontage] = useState(false) const [montageSelectionError, setMontageSelectionError] = useState('') const [comicOpenError, setComicOpenError] = useState('') + const [settingsError, setSettingsError] = useState('') + const [settingsBusy, setSettingsBusy] = useState(false) + const settingsPending = useRef(false) const moveRef = useRef(null) const itemRef = useRef(null) const videoRef = useRef(null) @@ -302,15 +305,23 @@ export function MediaFeedItem({ file, index, isActive, onVisible, onMeasured, ma setSelectedOutput(index) }, [file, index, isComic, isScene, setMediaFilter, setSelectedOutput]) - const handleLoadSettings = useCallback(() => { + const handleOutputSettings = useCallback(async (reroll: boolean) => { + if (settingsPending.current) return + settingsPending.current = true + setSettingsBusy(true) + setSettingsError('') setSelectedOutput(index) - setTimeout(() => loadSettingsFromOutput(), 50) - }, [index, setSelectedOutput, loadSettingsFromOutput]) - - const handleReroll = useCallback(() => { - setSelectedOutput(index) - setTimeout(() => rerollGeneration(), 50) - }, [index, setSelectedOutput, rerollGeneration]) + try { + const action = reroll ? rerollGeneration : loadSettingsFromOutput + const restored = await action({ name: file.name, workspace: outputWorkspace }) + if (restored === false) setSettingsError(t('settingsUnavailable')) + } catch (error) { + setSettingsError(error instanceof Error ? error.message : t('settingsUnavailable')) + } finally { + settingsPending.current = false + setSettingsBusy(false) + } + }, [file.name, outputWorkspace, index, setSelectedOutput, loadSettingsFromOutput, rerollGeneration, t]) const handleUseAsEditorReplacement = useCallback(() => { const target = readVideoEditorReplacementTarget() @@ -811,19 +822,22 @@ export function MediaFeedItem({ file, index, isActive, onVisible, onMeasured, ma + {settingsError && {settingsError}} {file.type === 'video' && ( <> + + )}

{t('sfx.hint')}

{/* Duration (shown when no video — max 20s, MMAudio single-pass limit) */} - {!videoFilename && ( + {!videoGuide && (
diff --git a/ui/src/components/Sidebar/Sidebar.tsx b/ui/src/components/Sidebar/Sidebar.tsx index 8ae61ab6a..2648e6fe0 100644 --- a/ui/src/components/Sidebar/Sidebar.tsx +++ b/ui/src/components/Sidebar/Sidebar.tsx @@ -28,7 +28,6 @@ import { WangpModelControls } from './WangpModelControls' import { BlendControls } from './BlendControls' import { AnchorReturnBanner } from './AnchorReturnBanner' import { VoiceRefSection } from './VoiceRefSection' -import { ToolsPanel } from './ToolsPanel' import { Hunyuan3DPanel } from './Hunyuan3DPanel' import { HardwareStatusBar } from './HardwareStatusBar' import { H3PromptControls } from './H3PromptControls' @@ -37,12 +36,14 @@ import { PanoramaLoopPanel } from './PanoramaLoopPanel' import { BrandIdentity } from '../BrandIdentity' import { DirectorChat } from './DirectorChat' import { useUiTranslation } from '../../i18n' +import { StudioCommandPanels } from '../../features/studio/StudioCommandPanels' const ViggleControls = lazy(() => import('./ViggleControls').then(module => ({ default: module.ViggleControls }))) -const StudioImageCommandPanel = lazy(() => import('../../features/studio/StudioImageCommandPanel').then(module => ({ default: module.StudioImageCommandPanel }))) +const ToolsPanel = lazy(() => import('./ToolsPanel').then(module => ({ default: module.ToolsPanel }))) export function Sidebar() { const { t } = useUiTranslation('navigation') + const { t: tCommon } = useUiTranslation('common') const [toolsCollapsed, setToolsCollapsed] = useState(() => window.localStorage.getItem('hocuspocus-tools-sidebar-collapsed') === 'true') const generationMode = useStore(s => s.generationMode) @@ -104,8 +105,17 @@ export function Sidebar() { window.localStorage.setItem('hocuspocus-tools-sidebar-collapsed', 'false') setSidebarOpen(true) } + const openSpeechSubmission = () => { + setToolsCollapsed(false) + window.localStorage.setItem('hocuspocus-tools-sidebar-collapsed', 'false') + setSidebarOpen(true) + } window.addEventListener('hocuspocus:studio-image-open', openImageSubmission) - return () => window.removeEventListener('hocuspocus:studio-image-open', openImageSubmission) + window.addEventListener('hocuspocus:studio-speech-open', openSpeechSubmission) + return () => { + window.removeEventListener('hocuspocus:studio-image-open', openImageSubmission) + window.removeEventListener('hocuspocus:studio-speech-open', openSpeechSubmission) + } }, [setSidebarOpen]) useEffect(() => { @@ -198,7 +208,9 @@ export function Sidebar() {
{/* Tools mode: standalone post-processing (upscale / revoice) on any existing clip. Renders in place of the generation controls. */} - {isTools ? : isModel3d ? : ( + {isTools ? {tCommon('status.loading')}
}> + + : isModel3d ? : ( <> {/* Edit mode: sub-mode toggle + sub-controls */} {isEdit && } @@ -248,13 +260,9 @@ export function Sidebar() { {/* Prompt area (non-edit modes, skip for SFX/Mixer/Music which have their own UI) */} {!isEdit && !(isAudio && (audioSubMode === 'sfx' || audioSubMode === 'mixer' || audioSubMode === 'music')) && (isMultiClip ? : )} - {isImage && { - await useStore.getState().reconnectJobs() - if (useStore.getState().activeWorkspace === receipt.result.workspace) { - await useStore.getState().maybeRefreshGallery() - } - }} />} + {/* Video: reference images below prompt. In Frames mode the InputsPanel renders them as ordered tiles instead. */} diff --git a/ui/src/components/Sidebar/ToolsPanel.tsx b/ui/src/components/Sidebar/ToolsPanel.tsx index cee9ec605..43b3c3f04 100644 --- a/ui/src/components/Sidebar/ToolsPanel.tsx +++ b/ui/src/components/Sidebar/ToolsPanel.tsx @@ -1,4 +1,4 @@ -import { useEffect, useMemo, useState } from 'react' +import { lazy, Suspense, useEffect, useMemo, useState } from 'react' import { Wrench, Play } from 'lucide-react' import { useStore } from '../../stores/useStore' import { useUiTranslation } from '../../i18n' @@ -9,6 +9,11 @@ import { catalogItemToOutput, voiceRefFromOutput } from '../../features/asset-pi import { ToolsParamsPanel } from './ToolsParamsPanel' import { resolveToolSource } from '../../lib/toolSource' import { ToolsSourcePanel } from './ToolsSourcePanel' +import { canonicalToolsSource } from '../../features/studio/toolsSource' + +const ToolsCommandPanel = lazy(() => import('../../features/studio/ToolsCommandPanel').then(module => ({ + default: module.ToolsCommandPanel, +}))) export function ToolsPanel() { const { t } = useUiTranslation('studio') @@ -113,6 +118,18 @@ export function ToolsPanel() { (tool === 'revoice' && hasVideoSource && hasRefs) || (tool === 'remove_background' && hasImageSource) const flashvsrOff = flashvsrMode === 0 && method.startsWith('flashvsr') + const commandSource = (() => { + if (sourceAssetId) return sourceAssetId + if (!sourcePath && !sourceUrl) return '' + try { + return canonicalToolsSource(sourcePath, sourceUrl, sourceWorkspace, activeWorkspace) + } catch { + // The command panel displays the malformed value so the durable builder + // can reject it with a field-specific error instead of sending legacy + // paths to the old Tools endpoint. + return sourceUrl || sourcePath || '' + } + })() return (
@@ -173,6 +190,23 @@ export function ToolsPanel() { setRemoveBackgroundInstruction={setRemoveBackgroundInstruction} /> + + { + await useStore.getState().reconnectJobs() + if (useStore.getState().activeWorkspace === receipt.result.workspace) { + await useStore.getState().maybeRefreshGallery() + } + }} + /> + + {/* Run */}