feat(studio): share native audio and upscale commands

app/_launch_runtime.py

@@ -10672,6 +10672,13 @@ def _run_generation_with_preparation(job_id: str) -> bool:
     job = _jobs.get(job_id)
     if not isinstance(job, dict):
         return False
+    try:
+        native_worker = _image_generation_commands.native_worker(job)
+    except HTTPException as error:
+        finish_job(job, "failed", error=str(error.detail), message="Command recovery could not be verified")
+        return False
+    if native_worker is not None:
+        return bool(native_worker(job_id))
     params = job.get("params") if isinstance(job.get("params"), dict) else {}
     pending = params.get("_h3_window_plan_pending")
     if not isinstance(pending, dict):
@@ -10870,7 +10877,9 @@ async def generate(request: Request):
         prepare_generation_inputs(body, _generation_model_def, requested_workspace,
                                   uploads_dir=os.path.join(os.getcwd(), "uploads"),
                                   workspace_dir=_workspace_dir(requested_workspace),
-                                  prepared_images=getattr(request, "prepared_studio_images", False) is True)
+                                  prepared_images=getattr(request, "prepared_studio_images", False) is True,
+                                  prepared_speech=(getattr(request, "prepared_studio_speech", False) is True
+                                                   or getattr(request, "prepared_studio_audio", False) is True))
     except ValueError as error:
         raise HTTPException(status_code=400, detail=str(error)) from error
     try:
@@ -22438,6 +22447,13 @@ def _run_sfx_generation(job: dict, raw_params: dict, start_time: float):
         ):
             return False
 
+        from services.studio_sfx_execution import prepared_sfx_execution
+        from services.studio_sfx_commands import check_sfx_models
+        typed_sfx = prepared_sfx_execution(
+            job, raw_params, registry=_task_registry(job["workspace"]),
+            check_models=lambda variant: check_sfx_models(globals(), variant),
+        )
+
         out_dir = job.get("out_dir") or wgp.save_path
         os.makedirs(out_dir, exist_ok=True)
         wgp.save_path = out_dir
@@ -22459,11 +22475,17 @@ def _run_sfx_generation(job: dict, raw_params: dict, start_time: float):
                 video_path = candidate
 
         if video_path and not os.path.isfile(video_path):
+            if typed_sfx:
+                raise ValueError("The admitted SFX video is no longer available")
             print(f"[SFX] Warning: video_guide not found: {video_path}, falling back to text-only")
             video_path = None
 
         # If video provided, derive duration from it
-        if video_path:
+        if typed_sfx:
+            # The admission records the inspected guide duration and exact
+            # source identity. Do not rederive it or silently cap the request.
+            pass
+        elif video_path:
             try:
                 import decord
                 vr = decord.VideoReader(video_path)
@@ -22494,12 +22516,14 @@ def _run_sfx_generation(job: dict, raw_params: dict, start_time: float):
             )
             return False
 
-        # Download model files if needed
-        if not update_job(
-            job, message="Downloading MMAudio models...", phase="Downloading models",
-        ):
-            return False
-        wgp.download_mmaudio(variant_override=variant)
+        # Typed commands require installed dependencies, checked again above.
+        # Keep legacy provisioning until its callers have been migrated.
+        if not typed_sfx:
+            if not update_job(
+                job, message="Downloading MMAudio models...", phase="Downloading models",
+            ):
+                return False
+            wgp.download_mmaudio(variant_override=variant)
         if is_cancel_requested(job):
             return False
 
@@ -22552,10 +22576,10 @@ def _run_sfx_generation(job: dict, raw_params: dict, start_time: float):
         elapsed = time.time() - start_time
         for fname in new_files:
             ext = os.path.splitext(fname)[1].lower()
-            if ext not in {".wav", ".mp3", ".flac"}:
+            if ext not in {".wav", ".mp3", ".flac", ".mp4"}:
                 continue
             sidecar = {
-                "params": {
+                "params": copy.deepcopy(raw_params) if typed_sfx else {
                     "prompt": prompt,
                     "MMAudio_prompt": prompt,
                     "MMAudio_neg_prompt": neg_prompt,
@@ -23435,6 +23459,17 @@ async def tools_upscale(request: Request):
         "workspace": workspace, "out_dir": output_dir,
         "provenance": provenance,
     }
+    admit_command = getattr(request, "admit_generation_command", None)
+    if callable(admit_command):
+        # Only the typed in-process command adapter can transfer admission.
+        # Preserve the existing tool's resolved inputs and native worker;
+        # canonical task/receipt persistence now owns its queue lifecycle.
+        job["params"].pop("_non_durable_tool", None)
+        provenance["capability"] = "tools.upscale"
+        command_collection = (body.get("provenance") or {}).get("workspace_id")
+        if command_collection is not None:
+            provenance["workspace_id"] = command_collection
+        return admit_command(job["params"], workspace, provenance)
     _register_manual_generation_job(job)
     worker = _run_generation if execution_mode.policy().simulated else _run_tool_upscale
     threading.Thread(target=worker, args=(job_id,), daemon=False).start()
@@ -36223,7 +36258,7 @@ def _generation_task_fields(job: dict) -> dict:
     }.get(mode, "Generation job")
     if str(provenance.get("capability") or "") == "remove_background":
         task_title = "Tools · Remove background"
-    elif str(provenance.get("capability") or "") == "upscale":
+    elif str(provenance.get("capability") or "") in {"upscale", "tools.upscale"}:
         task_title = "Tools · Upscale"
     elif str(provenance.get("capability") or "") == "revoice":
         task_title = "Tools · Revoice"
@@ -36863,7 +36898,8 @@ def _classic_redirect():
               "generate": generate, "recast": recast_endpoint, "upscale": tools_upscale,
               **wangp_agent_handlers(api), **image_command_handlers(_image_generation_commands)},
     journal_path=os.path.join(os.path.dirname(__file__), "settings", "wangp-mcp-requests.sqlite3"),
-    command_operations=[*workspace_command_catalog()["operations"], *image_command_catalog()],
+    command_operations=[*workspace_command_catalog()["operations"], *image_command_catalog(
+        adapter.catalog for adapter in _image_generation_commands.operations.values())],
 ))
 
 # ============================================================================

app/routers/image_generation_commands.py

@@ -1,5 +1,6 @@
 """HTTP and MCP projections of the executable image command contract."""
 from __future__ import annotations
+from typing import Literal
 
 from fastapi import APIRouter, Request
 from pydantic import BaseModel, ConfigDict, Field, StrictStr, ValidationError, field_validator
@@ -11,6 +12,7 @@
 class ReferenceResolutionInput(BaseModel):
     model_config = ConfigDict(extra="forbid", strict=True)
     references: list[StrictStr] = Field(min_length=1, max_length=64)
+    media_kind: Literal["image", "audio", "video"] = "image"
 
 
 class UISubmissionContext(BaseModel):
@@ -37,7 +39,7 @@ def _ui_context(request):
         raise command_error(422, "invalid_ui_context", "Use only exact workflowId and runId attribution") from error
 
 
-def image_command_catalog():
+def image_command_catalog(additional_operations=()):
     spec = image_generation_schema()
     studio = studio_image_schema()
     studio_input = dict(studio["input"])
@@ -60,18 +62,20 @@ def image_command_catalog():
              "description": "Admit an image job with an installed model and explicit output workspace. Version 1 is a single text-to-image request; version 2 accepts the complete typed Studio image parameters, canonical references, LoRAs and image processors. Preserve literal prompts and reuse intent_id only for retries. The receipt proves admission; inspect its task for completion.",
              "inputSchema": envelope},
             {"name": "generation.receipt", "version": 1, "domain": "studio", "mutation": False,
-             "description": "Read an immutable image admission and its current canonical task in the exact original output workspace.",
+             "description": "Read an immutable generation admission and its current canonical task in the exact original output workspace.",
              "inputSchema": {"type": "object", "additionalProperties": False,
                              "properties": {"version": {"type": "integer", "const": 1},
                                             "operation": {"const": "generation.receipt"}, "input": receipt_input},
-                             "required": ["version", "operation", "input"]}}]
+                             "required": ["version", "operation", "input"]}}, *additional_operations]
 
 
 def image_command_handlers(service):
-    async def submit(arguments):
-        if not isinstance(arguments, dict) or set(arguments) != {"version", "intent_id", "input"}:
-            raise command_error(422, "invalid_command", "Use version, intent_id and input for the image tool")
-        return await service.submit({**arguments, "operation": "generation.image"}, trusted_tool="external_agent")
+    def submission_handler(operation):
+        async def submit(arguments):
+            if not isinstance(arguments, dict) or set(arguments) != {"version", "intent_id", "input"}:
+                raise command_error(422, "invalid_command", "Use version, intent_id and input for the generation tool")
+            return await service.submit({**arguments, "operation": operation}, trusted_tool="external_agent")
+        return submit
 
     def receipt(arguments):
         if (not isinstance(arguments, dict) or set(arguments) != {"version", "input"}
@@ -81,15 +85,17 @@ def receipt(arguments):
             raise command_error(422, "invalid_command", "Use version 1 with workspace and intent_id")
         return service.receipt(**arguments["input"])
 
-    return {"generation.image": submit, "generation.receipt": receipt}
+    operations = {"generation.image", *getattr(service, "operations", {})}
+    return {**{operation: submission_handler(operation) for operation in operations}, "generation.receipt": receipt}
 
 
 def create_image_generation_commands_router(service):
     router = APIRouter()
 
     @router.get("/api/v1/generation/commands")
     def catalog():
-        return {"version": 2, "operations": image_command_catalog()}
+        return {"version": 2, "operations": image_command_catalog(
+            adapter.catalog for adapter in getattr(service, "operations", {}).values())}
 
     @router.post("/api/v1/generation/commands")
     async def submit(request: Request):
@@ -118,7 +124,8 @@ def references(body: ReferenceResolutionInput):
         if any(not 1 <= len(value) <= 8192 for value in body.references):
             raise command_error(422, "invalid_reference", "An exact bounded media reference is required")
         try:
-            return {"references": [resolve(value) for value in body.references]}
+            return {"references": [resolve(value) if body.media_kind == "image"
+                                   else resolve(value, media_kind=body.media_kind) for value in body.references]}
         except (ValueError, OSError) as error:
             raise command_error(422, "invalid_reference", str(error)) from error
 

app/routers/studio_music_commands.py

@@ -0,0 +1,40 @@
+"""Serializable catalog for the local Studio music operation."""
+
+from services.studio_music_spec import studio_music_schema
+
+
+def music_command_catalog():
+    """Return the exact closed ``generation.music`` discovery entry."""
+    schema = studio_music_schema()
+    input_schema = dict(schema["input"])
+    definitions = input_schema.pop("$defs", {})
+    return {
+        "name": "generation.music",
+        "version": 2,
+        "supportedVersions": [2],
+        "domain": "studio",
+        "mutation": True,
+        "description": (
+            "Admit one local music generation with literal lyrics and caption "
+            "through the canonical generation queue. The selected ACE-Step or "
+            "MiniMax-Music3 model must already be installed; retries reuse the "
+            "same intent receipt."
+        ),
+        "musicModelTypes": list(schema["music_model_types"]),
+        "guideRevision": schema["guide_revision"],
+        "inputSchema": {
+            "type": "object",
+            "additionalProperties": False,
+            "$defs": definitions,
+            "properties": {
+                "version": {"type": "integer", "const": 2},
+                "operation": {"const": "generation.music"},
+                "intent_id": schema["intent_id"],
+                "input": input_schema,
+            },
+            "required": ["version", "operation", "intent_id", "input"],
+        },
+    }
+
+
+__all__ = ["music_command_catalog"]

app/routers/studio_sfx_commands.py

@@ -0,0 +1,20 @@
+"""Executable SFX command schema shared by local HTTP and external MCP."""
+from services.studio_sfx_spec import studio_sfx_schema
+
+
+def sfx_command_catalog():
+    schema = studio_sfx_schema()
+    input_schema = dict(schema["input"])
+    definitions = input_schema.pop("$defs", {})
+    return {
+        "name": "generation.sfx", "version": 2, "supportedVersions": [2],
+        "domain": "studio", "mutation": True,
+        "description": "Generate sound effects with installed MMAudio files in an explicit output workspace. Preserve literal prompts. Text-only requests produce audio with a duration up to 20 seconds; video-guided requests use the inspected video duration and produce a video with new audio. Sources require canonical references. Reuse intent_id only to recover an existing admission; follow its task for completion.",
+        "inputSchema": {
+            "type": "object", "additionalProperties": False, "$defs": definitions,
+            "properties": {"version": {"type": "integer", "const": 2},
+                           "operation": {"const": "generation.sfx"},
+                           "intent_id": schema["intent_id"], "input": input_schema},
+            "required": ["version", "operation", "intent_id", "input"],
+        },
+    }

app/routers/studio_speech_commands.py

@@ -0,0 +1,20 @@
+"""Serializable speech operation projected through the shared HTTP/MCP router."""
+from services.studio_speech_spec import studio_speech_schema
+
+
+def speech_command_catalog():
+    schema = studio_speech_schema()
+    input_schema = dict(schema["input"])
+    definitions = input_schema.pop("$defs", {})
+    return {
+        "name": "generation.speech", "version": 2, "supportedVersions": [2],
+        "domain": "studio", "mutation": True,
+        "description": "Admit speech using an installed speech model, literal text, voice settings and canonical audio references in an explicit output workspace. Preserve the original speaker text separately from the effective native prompt. Reuse intent_id only for retries; the receipt proves admission, and its task reports completion.",
+        "inputSchema": {
+            "type": "object", "additionalProperties": False, "$defs": definitions,
+            "properties": {"version": {"type": "integer", "const": 2},
+                           "operation": {"const": "generation.speech"},
+                           "intent_id": schema["intent_id"], "input": input_schema},
+            "required": ["version", "operation", "intent_id", "input"],
+        },
+    }

app/routers/tools_upscale_commands.py

@@ -0,0 +1,51 @@
+"""Pure catalog projection for the typed Tools upscale operation."""
+
+from __future__ import annotations
+
+from copy import deepcopy
+
+from services.tools_upscale_spec import tools_upscale_schema
+
+
+def tools_upscale_command_catalog() -> dict:
+    """Return the exact operation entry consumed by HTTP/MCP discovery."""
+    schema = tools_upscale_schema()
+    input_schema = dict(schema["input"])
+    definitions = input_schema.pop("$defs", {})
+    return {
+        "name": "tools.upscale",
+        "version": 2,
+        "supportedVersions": [2],
+        "domain": "tools",
+        "mutation": True,
+        "description": (
+            "Upscale one exact image or video source in an explicit output "
+            "workspace with an installed local processor. Preserve the "
+            "source workspace, processor settings and intent_id; the shared "
+            "receipt proves admission and its canonical task reports completion."
+        ),
+        "inputSchema": {
+            "type": "object",
+            "additionalProperties": False,
+            "$defs": definitions,
+            "properties": {
+                "version": {"type": "integer", "const": 2},
+                "operation": {"const": "tools.upscale"},
+                "intent_id": deepcopy(schema["intent_id"]),
+                "input": input_schema,
+            },
+            "required": ["version", "operation", "intent_id", "input"],
+        },
+    }
+
+
+# Short aliases used by catalog exporters in adjacent command slices.
+tools_upscale_catalog = tools_upscale_command_catalog
+upscale_command_catalog = tools_upscale_command_catalog
+
+
+__all__ = [
+    "tools_upscale_catalog",
+    "tools_upscale_command_catalog",
+    "upscale_command_catalog",
+]