From 41494d447d23b231701f4d4429bad5dfb80889f1 Mon Sep 17 00:00:00 2001 From: IAnMove <216241348+IAnMove@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:10:36 +0200 Subject: [PATCH 01/59] Constrain speech prompts and split validation helpers --- app/services/studio_speech_preparation.py | 510 +++++++++++++++++ app/services/studio_speech_spec.py | 637 +++++++++++++++++++++ docs/development/SPEECH_COMMANDS.md | 73 +++ tests/test_studio_speech_runtime_review.py | 195 +++++++ tests/test_studio_speech_spec.py | 246 ++++++++ 5 files changed, 1661 insertions(+) create mode 100644 app/services/studio_speech_preparation.py create mode 100644 app/services/studio_speech_spec.py create mode 100644 docs/development/SPEECH_COMMANDS.md create mode 100644 tests/test_studio_speech_runtime_review.py create mode 100644 tests/test_studio_speech_spec.py diff --git a/app/services/studio_speech_preparation.py b/app/services/studio_speech_preparation.py new file mode 100644 index 000000000..5ff2738b1 --- /dev/null +++ b/app/services/studio_speech_preparation.py @@ -0,0 +1,510 @@ +"""Provider-free preparation for the closed Studio speech command. + +The preparation boundary validates only facts declared by the selected native +handler, checks that speech references and settings are coherent, and asks the +existing ``StudioSpeechResources`` object to inspect canonical media/LoRAs. +It returns detached native parameters and resource identities; it does not +load a model, schedule a worker or create a second queue. +""" + +from __future__ import annotations + +from copy import deepcopy +import math +import re +from typing import Any, Mapping + +from fastapi import HTTPException + +from services.image_generation_commands import command_error +from services.studio_image_resources import validate_lora_multipliers +from services.studio_speech_spec import SPEECH_MODEL_TYPES + + +_AUDIO_REFERENCE_FIELDS = tuple( + ["audio_guide"] + [f"audio_guide{index}" for index in range(2, 7)] +) +_AUDIO_MODE_FLAGS = frozenset({"N", "V"}) + + +def _definition_for(model_definition, model_type: str) -> dict[str, Any]: + if callable(model_definition): + definition = model_definition(model_type) + elif isinstance(model_definition, Mapping): + definition = model_definition.get(model_type) + else: + definition = None + if not isinstance(definition, dict): + raise ValueError("Choose an installed speech model from the model catalog") + return deepcopy(definition) + + +def _finite_number(value, field: str, *, minimum: float | None = None, + maximum: float | None = None) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"input.params.{field} must be a finite number") + parsed = float(value) + if not math.isfinite(parsed): + raise ValueError(f"input.params.{field} must be a finite number") + if minimum is not None and parsed < minimum: + raise ValueError(f"input.params.{field} is below the model's declared minimum") + if maximum is not None and parsed > maximum: + raise ValueError(f"input.params.{field} exceeds the model's declared maximum") + return parsed + + +def _choice_values(choices) -> list[str]: + if not isinstance(choices, (list, tuple)): + return [] + values: list[str] = [] + for item in choices: + if isinstance(item, Mapping): + value = item.get("value") + elif isinstance(item, (list, tuple)) and len(item) >= 2: + value = item[1] + else: + value = item + if isinstance(value, str): + values.append(value) + return values + + +def _model_mode(working: dict[str, Any], definition: dict[str, Any]) -> None: + mode = working.get("model_mode") + if mode is not None and not isinstance(mode, str): + raise ValueError("input.params.model_mode must be a string or null") + modes = definition.get("model_modes") + if not isinstance(modes, Mapping): + if mode not in (None, ""): + raise ValueError("input.params.model_mode is not supported by this speech model") + return + + choices = _choice_values(modes.get("choices")) + default = modes.get("default") + if mode in (None, ""): + if isinstance(default, str) and default != "": + if choices and default not in choices: + raise ValueError("the speech model advertises an invalid model_mode default") + working["model_mode"] = default + return + if choices and mode not in choices: + raise ValueError("input.params.model_mode is not one of the model's declared choices") + + +def _duration(working: dict[str, Any], definition: dict[str, Any]) -> None: + value = working.get("duration_seconds") + slider = definition.get("duration_slider") + if not isinstance(slider, Mapping): + if value is not None: + _finite_number(value, "duration_seconds", minimum=0) + return + if value is None: + # ``default=0`` is meaningful for DramaBox (auto duration), so test + # key presence rather than using truthiness. + for key in ("default", "max"): + if key in slider and slider[key] is not None: + value = slider[key] + break + if value is None: + value = 600 + working["duration_seconds"] = value + minimum = slider.get("min") + maximum = slider.get("max") + _finite_number(value, "duration_seconds", minimum=float(minimum) if minimum is not None else 0, + maximum=float(maximum) if maximum is not None else None) + + +def _validate_inference_steps(steps: Any, definition: dict[str, Any]) -> None: + if type(steps) is not int or steps < 0: + raise ValueError("input.params.num_inference_steps must be a non-negative integer") + # Most speech handlers expose no step control and use the zero sentinel. + # Scenema is the declared exception: its locked native profile forces its + # own fixed step count (currently eight) even though the control is not + # user-editable. Preserve that model-owned value for the native facade. + if definition.get("inference_steps") is False and steps != 0 and not definition.get("lock_inference_steps"): + raise ValueError("input.params.num_inference_steps must be zero for this speech model") + if definition.get("inference_steps") is not False: + lower = definition.get("inference_steps_min") + upper = definition.get("inference_steps_max") + if lower is not None and steps < int(lower): + raise ValueError("input.params.num_inference_steps is below the model's declared minimum") + if upper is not None and steps > int(upper): + raise ValueError("input.params.num_inference_steps exceeds the model's declared maximum") + + +def _sampling_phases(working: dict[str, Any], definition: dict[str, Any]): + if "guidance_phases" not in working: + declared_phases = definition.get("guidance_max_phases") + working["guidance_phases"] = int(declared_phases) if declared_phases is not None else 1 + phases = working.get("guidance_phases", 1) + if type(phases) is not int or not 0 <= phases <= 16: + raise ValueError("input.params.guidance_phases must be an integer from 0 to 16") + maximum = definition.get("guidance_max_phases") + if maximum is not None and phases > int(maximum): + raise ValueError("input.params.guidance_phases exceeds the model's declared maximum") + return maximum + + +def _validate_sampling_scalars(working: dict[str, Any], definition: dict[str, Any]) -> None: + solver = working.get("sample_solver", "") + choices = _choice_values(definition.get("sample_solvers")) + if solver and choices and solver not in choices: + raise ValueError("input.params.sample_solver is not one of the model's declared choices") + + if working.get("negative_prompt") and definition.get("no_negative_prompt"): + raise ValueError("input.params.negative_prompt is not supported by this speech model") + temperature = working.get("temperature") + if temperature is not None: + _finite_number(temperature, "temperature", minimum=0) + if definition.get("temperature") is False: + raise ValueError("input.params.temperature is locked for this speech model") + pause = working.get("pause_seconds") + if pause is not None: + _finite_number(pause, "pause_seconds", minimum=0, maximum=2) + if not definition.get("pause_between_sentences"): + raise ValueError("input.params.pause_seconds is not supported by this speech model") + + +def _validate_top_sampling_field( + field: str, value: Any, capability: str, definition: dict[str, Any] +) -> None: + if value is None: + return + if not definition.get(capability): + raise ValueError(f"input.params.{field} is not supported by this speech model") + if field == "top_p": + _finite_number(value, field, minimum=0, maximum=1) + elif type(value) is not int or value < 0: + raise ValueError("input.params.top_k must be a non-negative integer") + + +def _validate_sampling_options(working: dict[str, Any], definition: dict[str, Any]) -> None: + _validate_sampling_scalars(working, definition) + for field, capability in (("top_p", "top_p_slider"), ("top_k", "top_k_slider")): + _validate_top_sampling_field(field, working.get(field), capability, definition) + + +def _sampling(working: dict[str, Any], definition: dict[str, Any]) -> int: + steps = working.get("num_inference_steps", 0) + working.setdefault("num_inference_steps", 0) + _validate_inference_steps(steps, definition) + maximum = _sampling_phases(working, definition) + _validate_sampling_options(working, definition) + return max(1, int(maximum if maximum is not None else 1)) + + +def _strip_audio_modifier_flags( + mode: str, source: Mapping[str, Any] | None, + definition: Mapping[str, Any] | None, +) -> str: + custom_flags = source.get("custom_flags") if source else None + if isinstance(custom_flags, Mapping): + for flag in custom_flags: + if isinstance(flag, str) and flag: + mode = mode.replace(flag.upper(), "") + custom = definition.get("audio_prompt_type_custom_option") if definition else None + custom_flag = custom.get("flag") if isinstance(custom, Mapping) else None + if isinstance(custom_flag, str) and custom_flag: + mode = mode.replace(custom_flag.upper(), "") + return mode + + +def _base_audio_mode(value: str, source: Mapping[str, Any] | None, + definition: Mapping[str, Any] | None = None) -> str: + mode = value.upper() + mode_without_modifiers = "".join(char for char in mode if char not in _AUDIO_MODE_FLAGS) + # A mode already present in the native selection is a complete value + # (Scenema's A2/AB2 include the SeedVC ``2`` flag). Strip custom flags + # only when they are being used as standalone modifiers. + selections = _choice_values(source.get("selection")) if source else [] + if mode_without_modifiers in selections: + return mode_without_modifiers + mode = _strip_audio_modifier_flags(mode_without_modifiers, source, definition) + return "".join(char for char in mode if char not in _AUDIO_MODE_FLAGS) + + +def _voice_mode_for_count(count: int, selection: list[str]) -> str: + if not selection: + return "" + if count <= 0: + return selection[0] + if count == 1: + return selection[min(1, len(selection) - 1)] + return selection[min(2, len(selection) - 1)] + + +def _audio_mode_source( + definition: dict[str, Any], +) -> tuple[Mapping[str, Any] | None, list[str]]: + source = definition.get("audio_prompt_type_sources") + source = source if isinstance(source, Mapping) else None + selection = _choice_values(source.get("selection")) if source else [] + # Some model definitions use a plain string list for selection and the + # helper above deliberately handles only native list/tuple values. + if source and not selection and isinstance(source.get("selection"), list): + selection = [item for item in source["selection"] if isinstance(item, str)] + if source and not selection: + raise ValueError("the speech model advertises no audio prompt choices") + return source, selection + + +def _audio_mode_default( + working: dict[str, Any], raw_mode: str, source: Mapping[str, Any] | None +) -> str: + mode = raw_mode + if not mode and source and isinstance(source.get("default"), str): + mode = source["default"] + if mode: + working["audio_prompt_type"] = mode + return mode + + +def _speech_audio_mode( + working: dict[str, Any], definition: dict[str, Any] +) -> tuple[str, str, Mapping[str, Any] | None, list[str]]: + raw_mode = working.get("audio_prompt_type") or "" + if not isinstance(raw_mode, str): + raise ValueError("input.params.audio_prompt_type must be a string") + source, selection = _audio_mode_source(definition) + mode = _audio_mode_default(working, raw_mode, source) + base = _base_audio_mode(mode, source, definition) + return mode, base, source, selection + + +def _validate_declared_audio_mode( + mode: str, + base: str, + source: Mapping[str, Any] | None, + definition: dict[str, Any], + selection: list[str], +) -> None: + if source: + custom = definition.get("audio_prompt_type_custom_option") + custom_flag = custom.get("flag") if isinstance(custom, Mapping) else None + source_custom_flags = source.get("custom_flags") + has_source_custom = ( + isinstance(source_custom_flags, Mapping) + and any(isinstance(flag, str) and flag and flag.upper() in mode.upper() + for flag in source_custom_flags) + ) + if base not in selection: + # A custom-only flag (DramaBox's ``0``) has an empty base and is + # valid alongside the selected empty mode. + if not ( + not base + and ( + (isinstance(custom_flag, str) and custom_flag.upper() in mode.upper()) + or has_source_custom + ) + ): + raise ValueError("input.params.audio_prompt_type is not a declared model choice") + elif mode and mode not in {"A", "AN", "AV"}: + # Chatterbox exposes any_audio_prompt but no choice map; its native + # handler accepts its historical A selector and no other mode. + raise ValueError("input.params.audio_prompt_type is not supported by this speech model") + + +def _validate_voice_count( + working: dict[str, Any], definition: dict[str, Any], mode: str, + source: Mapping[str, Any] | None, selection: list[str], +) -> int: + count = working.get("_tts_voice_count", 0) + if type(count) is not int or not 0 <= count <= 6: + raise ValueError("input.params._tts_voice_count must be an integer from 0 to 6") + maximum_count = definition.get("max_voice_count") + if maximum_count is not None and count > int(maximum_count): + raise ValueError("input.params._tts_voice_count exceeds the model's voice limit") + + if definition.get("audio_mode_from_voice_count"): + expected = _voice_mode_for_count(count, selection) + if _base_audio_mode(mode, source, definition) != expected: + raise ValueError("input.params.audio_prompt_type must match the selected voice count") + return count + + +def _audio_reference_state( + working: dict[str, Any], definition: dict[str, Any], count: int +) -> tuple[dict[str, Any], int]: + refs = {field: working.get(field) for field in _AUDIO_REFERENCE_FIELDS} + highest_ref = 0 + for index, field in enumerate(_AUDIO_REFERENCE_FIELDS, start=1): + value = refs[field] + if value not in (None, ""): + highest_ref = index + if definition.get("audio_mode_from_voice_count") and count < index: + raise ValueError(f"input.params.{field} requires that voice slot to be selected") + if highest_ref and definition.get("audio_mode_from_voice_count") and count < highest_ref: + raise ValueError("audio references exceed the selected voice count") + return refs, highest_ref + + +def _validate_primary_audio_references(refs: dict[str, Any], base: str) -> None: + needs_first = "A" in base or base == "2" + needs_second = "B" in base or base == "2" + if needs_first and not refs["audio_guide"]: + raise ValueError("input.params.audio_prompt_type requires audio_guide") + if needs_second and not refs["audio_guide2"]: + raise ValueError("input.params.audio_prompt_type requires audio_guide2") + if refs["audio_guide"] and not needs_first: + raise ValueError("audio_guide is selected but audio_prompt_type has no first reference") + if refs["audio_guide2"] and not needs_second: + raise ValueError("audio_guide2 is selected but audio_prompt_type has no second reference") + + +def _validate_additional_audio_references(refs: dict[str, Any], base: str) -> None: + for index in range(3, 7): + if refs[f"audio_guide{index}"] and "B" not in base: + raise ValueError(f"audio_guide{index} requires a multi-voice audio_prompt_type") + + +def _validate_audio_reference_modes( + refs: dict[str, Any], base: str, source: Mapping[str, Any] | None +) -> None: + # Explicit model choices are the evidence that A/B references are active. + # Models without a choice map (Chatterbox) leave reference requirements to + # their native handler instead of this adapter inventing one. + if source: + _validate_primary_audio_references(refs, base) + _validate_additional_audio_references(refs, base) + + +def _validate_audio_prompts(working: dict[str, Any], definition: dict[str, Any]) -> None: + mode, base, source, selection = _speech_audio_mode(working, definition) + _validate_declared_audio_mode(mode, base, source, definition, selection) + count = _validate_voice_count(working, definition, mode, source, selection) + refs, _ = _audio_reference_state(working, definition, count) + _validate_audio_reference_modes(refs, base, source) + + +def _setting_id(setting: Mapping[str, Any], index: int) -> str: + explicit = setting.get("id") + if isinstance(explicit, str) and explicit.strip(): + return explicit.strip() + name = setting.get("name") + if isinstance(name, str) and name.strip(): + normalized = re.sub(r"[^a-z0-9_]+", "_", name.strip().lower()).strip("_") + if normalized: + return normalized + return f"custom_setting_{index + 1}" + + +def _custom_setting_definitions(definition: dict[str, Any]): + definitions = definition.get("custom_settings") + if not isinstance(definitions, list): + definitions = definition.get("custom_settings_def") + return definitions if isinstance(definitions, list) else None + + +def _validate_custom_setting_range(key: str, numeric: float | None, setting: Mapping[str, Any]) -> None: + if numeric is None: + return + lower = setting.get("min", setting.get("minimum")) + upper = setting.get("max", setting.get("maximum")) + if lower is not None and numeric < float(lower): + raise ValueError(f"input.params.custom_settings.{key} is below its declared minimum") + if upper is not None and numeric > float(upper): + raise ValueError(f"input.params.custom_settings.{key} exceeds its declared maximum") + + +def _validate_custom_setting(key: str, value: Any, setting: Mapping[str, Any]) -> None: + setting_type = str(setting.get("type", "text")).lower() + if value == "" and setting_type in {"int", "float"}: + # The Studio numeric control uses an empty string while a + # selectable setting is inactive; native handlers remove it. + return + if setting_type == "int": + if type(value) is not int: + raise ValueError(f"input.params.custom_settings.{key} must be an integer") + numeric = float(value) + elif setting_type == "float": + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"input.params.custom_settings.{key} must be a number") + numeric = float(value) + if not math.isfinite(numeric): + raise ValueError(f"input.params.custom_settings.{key} must be finite") + elif setting_type == "bool": + if type(value) is not bool: + raise ValueError(f"input.params.custom_settings.{key} must be boolean") + numeric = None + else: + if not isinstance(value, str): + raise ValueError(f"input.params.custom_settings.{key} must be text") + numeric = None + _validate_custom_setting_range(key, numeric, setting) + + +def _validate_custom_settings(working: dict[str, Any], definition: dict[str, Any]) -> None: + values = working.get("custom_settings") + if values is None: + return + if not isinstance(values, dict): + raise ValueError("input.params.custom_settings must be an object") + definitions = _custom_setting_definitions(definition) + if not isinstance(definitions, list): + if values: + raise ValueError("input.params.custom_settings is not supported by this speech model") + return + known = {_setting_id(item, index): item for index, item in enumerate(definitions) if isinstance(item, Mapping)} + unknown = sorted(set(values) - set(known)) + if unknown: + raise ValueError("input.params.custom_settings contains an unknown model setting") + for key, value in values.items(): + _validate_custom_setting(key, value, known[key]) + + +def _validate_loras(working: dict[str, Any], definition: dict[str, Any], phases: int) -> None: + names = working.get("activated_loras") or [] + if names and not definition.get("enabled_audio_lora"): + raise ValueError("selected LoRAs are not supported by this speech model") + if names or working.get("loras_multipliers"): + validate_lora_multipliers(working, phases) + + +def _validate_speech_model(model_type: str, definition: dict[str, Any], model_downloaded) -> None: + if model_type not in SPEECH_MODEL_TYPES: + raise ValueError("Choose a registered speech model; music and SFX models use another operation") + if definition.get("audio_only") is not True or definition.get("image_outputs"): + raise ValueError("The selected model is not an audio-only speech model") + if not model_downloaded(model_type): + raise command_error(409, "model_unavailable", "Required speech model files are not installed; install them before submitting") + + +def prepare_studio_speech(params, *, model_definition, model_downloaded, resources, + execution_policy): + """Return detached native speech parameters and inspected resources. + + ``params`` is normally the effective output of + :func:`freeze_studio_speech_spec`; accepting a mapping directly keeps this + boundary easy to test and lets the native facade supply its own snapshot. + """ + if not isinstance(params, dict): + raise command_error(422, "invalid_studio_speech_input", "Speech parameters must be an object") + working = deepcopy(params) + try: + workspace = working.get("workspace") + if not isinstance(workspace, str) or not workspace.strip(): + raise ValueError("input.workspace must be an explicit output workspace") + execution_policy(workspace) + model_type = working.get("model_type") + if not isinstance(model_type, str): + raise ValueError("input.params.model_type must be a string") + definition = _definition_for(model_definition, model_type) + _validate_speech_model(model_type, definition, model_downloaded) + phases = _sampling(working, definition) + _model_mode(working, definition) + _duration(working, definition) + _validate_audio_prompts(working, definition) + _validate_custom_settings(working, definition) + _validate_loras(working, definition, phases) + prepared, media = resources.prepare_media(working) + loras = resources.prepare_loras(working, definition) + if not isinstance(prepared, dict): + raise ValueError("speech resource preparation returned invalid native parameters") + return deepcopy(prepared), [*deepcopy(media), *deepcopy(loras)] + except HTTPException: + raise + except (OSError, ValueError) as error: + raise command_error(422, "invalid_studio_speech_input", str(error)) from error + + +__all__ = ["prepare_studio_speech"] diff --git a/app/services/studio_speech_spec.py b/app/services/studio_speech_spec.py new file mode 100644 index 000000000..85d8b3932 --- /dev/null +++ b/app/services/studio_speech_spec.py @@ -0,0 +1,637 @@ +"""Closed, provider-free contract for the Studio speech command. + +The browser has one native parameter map for all Studio modes. A speech +command must freeze the speech part of that map before model lookup, resource +inspection, task admission or worker effects. This module owns that boundary; +the model handler remains the authority for provider-specific generation +validation. + +The v2 envelope is:: + + { + "version": 2, + "operation": "generation.speech", + "intent_id": "...", + "input": { + "workspace": "...", + "params": {"prompt": "...", "model_type": "...", ...} + } + } + +``original`` is an immutable detached copy of the submitted envelope. The +``effective`` map contains only deterministic adapter defaults; model-derived +defaults are applied by the preparation boundary. The fingerprint covers +the effective operation, workspace and native parameters, while excluding +the transport intent and caller metadata. + +Audio references are syntax-checked here and resolved by +``StudioSpeechResources`` later. No filesystem path, model catalog, provider +or provenance authority is consulted in this module. +""" + +from __future__ import annotations + +from copy import deepcopy +import hashlib +import json +import re +from typing import Annotated, Any, Literal + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StringConstraints, + StrictBool, + StrictFloat, + StrictInt, + StrictStr, + ValidationError, + field_validator, + model_validator, +) + +from services.image_generation_spec import ImageGenerationSpecError +from services.studio_image_spec import _validate_reference + + +SCHEMA_VERSION = 2 +FINGERPRINT_VERSION = 2 +OPERATION = "generation.speech" + +_MAX_ID_LENGTH = 240 +_MAX_INTENT_LENGTH = 160 +_MAX_PROMPT_LENGTH = 200_000 +_MAX_LORA_COUNT = 64 +_MAX_VOICE_COUNT = 6 + +# These are adapter-owned values. They describe the native selectors that +# make an audio submission unambiguous. Duration, model mode and model +# custom settings stay model-owned and are filled during preflight. +STUDIO_SPEECH_DEFAULTS: dict[str, Any] = { + "generation_mode": "audio", + "_audio_sub_mode": "speech", + "video_length": 0, + "image_mode": 0, + "multi_prompts_gen_type": 2, + "negative_prompt": "", + "repeat_generation": 1, + "activated_loras": [], + "loras_multipliers": "", + "audio_prompt_type": "", + "prompt_enhancer": "", + "minimax_h3_turbo_mode": False, + "_tts_speaker_name1": "", + "_tts_speaker_name2": "", + "_tts_speaker_name3": "", + "_tts_speaker_name4": "", + "_tts_speaker_name5": "", + "_tts_speaker_name6": "", + "_tts_voice_count": 0, +} + +# The allowlist is intentionally explicit. Several music and sound-effect +# handlers advertise ``audio_only`` and the same ``tts`` family, so that +# metadata alone cannot safely turn this speech operation into a music/SFX +# operation. New handlers need a deliberate registration here. +SPEECH_MODEL_TYPES = frozenset( + { + "kugelaudio_0_open", + "qwen3_tts_customvoice", + "qwen3_tts_voicedesign", + "qwen3_tts_base", + "chatterbox", + "index_tts2", + "scenema_audio", + "dramabox_audio", + } +) + +INACTIVE_SPEECH_FIELDS = ( + "image_mode", + "video_length", + "minimax_h3_turbo_mode", + "image_start", + "image_end", + "image_refs", + "image_guide", + "image_mask", + "video_guide", + "video_mask", + "video_source", + "audio_source", + "MMAudio_setting", + "MMAudio_prompt", + "MMAudio_neg_prompt", + "h3_ref_videos", + "h3_ref_audios", + "minimax_h3_references", + "spatial_upsampling", + "temporal_upsampling", + "wangp_processor_settings", +) + +EXCLUDED_SPEECH_FIELDS = ( + "actor", + "client", + "permission", + "provenance", + "workspace inside input.params", + "filesystem paths", + "free-form provider payloads", + "music and SFX model types", + "video, avatar and model3d controls", +) + +SUPPORTED_INPUT_FIELDS = ( + "prompt", + "alt_prompt", + "model_type", + "resolution", + "video_length", + "num_inference_steps", + "guidance_scale", + "seed", + "image_mode", + "generation_mode", + "negative_prompt", + "repeat_generation", + "activated_loras", + "loras_multipliers", + "audio_prompt_type", + "audio_guide", + "audio_guide2", + "audio_guide3", + "audio_guide4", + "audio_guide5", + "audio_guide6", + "audio_source", + "_audio_sub_mode", + "_tts_original_prompt", + "_tts_speaker_name1", + "_tts_speaker_name2", + "_tts_speaker_name3", + "_tts_speaker_name4", + "_tts_speaker_name5", + "_tts_speaker_name6", + "_tts_voice_count", + "duration_seconds", + "pause_seconds", + "temperature", + "top_p", + "top_k", + "audio_scale", + "audio_guidance_scale", + "alt_scale", + "guidance_phases", + "flow_shift", + "sample_solver", + "settings_version", + "prompt_enhancer", + "model_mode", + "custom_settings", + "tts_dynaudnorm", + "tts_comp_threshold", + "tts_comp_attack", + "tts_comp_release", + "tts_comp_makeup", + "multi_prompts_gen_type", + "minimax_h3_turbo_mode", + "image_start", + "image_end", + "image_refs", + "image_guide", + "image_mask", + "video_guide", + "video_mask", + "video_source", + "spatial_upsampling", + "temporal_upsampling", + "wangp_processor_settings", + "MMAudio_setting", + "MMAudio_prompt", + "MMAudio_neg_prompt", + "h3_ref_videos", + "h3_ref_audios", + "minimax_h3_references", +) + + +class StudioSpeechSpecError(ImageGenerationSpecError): + """Validation error for the closed Studio speech envelope.""" + + +class _ClosedModel(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True, populate_by_name=False) + + +_Identity = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=_MAX_ID_LENGTH), +] +_Workspace = Annotated[ + StrictStr, + StringConstraints( + min_length=1, + max_length=_MAX_ID_LENGTH, + pattern=r"^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$", + ), +] +_WorkspaceCollectionId = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=200), +] +_IntentId = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=_MAX_INTENT_LENGTH), +] +_Prompt = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=_MAX_PROMPT_LENGTH), +] +_Text = Annotated[StrictStr, StringConstraints(max_length=_MAX_PROMPT_LENGTH)] +_ShortText = Annotated[StrictStr, StringConstraints(max_length=8192)] +_ModelMode = Annotated[StrictStr, StringConstraints(max_length=256)] +_AudioPromptType = Annotated[ + StrictStr, + StringConstraints(max_length=32, pattern=r"^[A-Za-z0-9]*$"), +] +_NonNegativeInt = Annotated[StrictInt, Field(ge=0, le=100_000)] +_Count = Annotated[StrictInt, Field(ge=1, le=100)] +_VoiceCount = Annotated[StrictInt, Field(ge=0, le=_MAX_VOICE_COUNT)] +_Seed = Annotated[StrictInt, Field(ge=-(2**63), le=2**63 - 1)] +_Finite = Annotated[StrictFloat, Field(allow_inf_nan=False)] +_CustomFloat = Annotated[StrictFloat, Field(allow_inf_nan=False)] +_NonNegativeFinite = Annotated[ + StrictFloat, + Field(ge=0, allow_inf_nan=False), +] +_Guidance = Annotated[ + StrictFloat, + Field(ge=0, le=1000, allow_inf_nan=False), +] +_Zero = Annotated[StrictInt, Field(ge=0, le=0)] +_DialogueMode = Annotated[StrictInt, Field(ge=2, le=2)] +_TtsDynaudnorm = Annotated[StrictInt, Field(ge=0, le=1)] | StrictBool + + +class StudioSpeechCustomSettings(_ClosedModel): + """Typed union of custom settings exposed by speech handlers. + + The selected model still owns which subset is valid and its metadata + ranges. Keeping the known IDs closed here prevents arbitrary nested JSON + from crossing the command boundary while allowing each speech handler's + currently published setting family. + """ + + auto_split_every_s: _CustomFloat | Literal[""] | None = None + exaggeration: _CustomFloat | None = None + pace: _CustomFloat | None = None + vc_steps: StrictInt | None = None + vc_cfg_rate: _CustomFloat | None = None + duration_multiplier: _CustomFloat | None = None + + +def _check_required_speech_texts(params: Any) -> None: + for field in ("prompt", "model_type", "resolution"): + if not getattr(params, field).strip(): + raise ValueError(f"{field} must contain a non-blank value") + + +def _check_speech_prompt_metadata(params: Any) -> None: + if params.tts_original_prompt is None and "tts_original_prompt" in params.model_fields_set: + raise ValueError("_tts_original_prompt must be a string when supplied") + + +def _check_speech_ranges(params: Any) -> None: + if params.pause_seconds is not None and params.pause_seconds > 2: + raise ValueError("pause_seconds must be between 0 and 2 seconds") + if params.temperature is not None and params.temperature > 2: + raise ValueError("temperature must be at most 2") + if params.tts_comp_threshold is not None and not -50 <= params.tts_comp_threshold <= -10: + raise ValueError("tts_comp_threshold must be between -50 and -10") + if params.tts_comp_release is not None and params.tts_comp_release > 500: + raise ValueError("tts_comp_release must be at most 500") + if params.tts_comp_makeup is not None and params.tts_comp_makeup > 12: + raise ValueError("tts_comp_makeup must be at most 12") + + +class StudioSpeechParams(_ClosedModel): + """Typed native speech parameters emitted by Studio.""" + + prompt: _Prompt + alt_prompt: _Text = "" + model_type: _Identity + resolution: Annotated[StrictStr, StringConstraints(min_length=1, max_length=128)] + + # Speech jobs retain the common native selectors as explicit inactive + # sentinels. They are exact zero values, not a mode inferred from a + # missing image field. + video_length: _Zero = 0 + num_inference_steps: _NonNegativeInt = 0 + guidance_scale: _Guidance = 1.0 + seed: _Seed = -1 + image_mode: _Zero = 0 + generation_mode: Literal["audio"] = "audio" + negative_prompt: _Text = "" + repeat_generation: _Count = 1 + activated_loras: list[Annotated[StrictStr, StringConstraints(min_length=1, max_length=8192)]] = Field( + default_factory=list, max_length=_MAX_LORA_COUNT + ) + loras_multipliers: _ShortText = "" + multi_prompts_gen_type: _DialogueMode = 2 + + # Native speech selectors and canonical reference URLs/asset IDs. Empty + # string and null are retained as inactive sentinels; paths are rejected + # by the validator below and resolved only by StudioSpeechResources. + audio_prompt_type: _AudioPromptType = "" + audio_guide: StrictStr | None = None + audio_guide2: StrictStr | None = None + audio_guide3: StrictStr | None = None + audio_guide4: StrictStr | None = None + audio_guide5: StrictStr | None = None + audio_guide6: StrictStr | None = None + audio_source: Literal["", None] = None + + # Studio keeps stale image/video controls in its shared state when the + # user switches to speech. They are accepted only as explicit inactive + # sentinels so a restored speech snapshot is not silently truncated. + image_start: Literal["", None] | list[StrictStr] = None + image_end: Literal["", None] | list[StrictStr] = None + image_refs: list[StrictStr] | None = None + image_guide: Literal["", None] | list[StrictStr] = None + image_mask: Literal["", None] | list[StrictStr] = None + video_guide: Literal["", None] = None + video_mask: Literal["", None] = None + video_source: Literal["", None] = None + spatial_upsampling: Literal["", None] = "" + temporal_upsampling: Literal["", None] = "" + wangp_processor_settings: dict[str, Any] | None = None + MMAudio_setting: _Zero | None = None + MMAudio_prompt: Literal["", None] = None + MMAudio_neg_prompt: Literal["", None] = None + h3_ref_videos: list[StrictStr] | None = None + h3_ref_audios: list[StrictStr] | None = None + minimax_h3_references: list[StrictStr] | None = None + + # Private UI fields are aliases because Pydantic field names cannot begin + # with an underscore. JSON serialization uses the native aliases. + audio_sub_mode: Literal["speech"] = Field("speech", alias="_audio_sub_mode") + tts_original_prompt: _Text | None = Field(None, alias="_tts_original_prompt") + tts_speaker_name1: _Text | None = Field("", alias="_tts_speaker_name1") + tts_speaker_name2: _Text | None = Field("", alias="_tts_speaker_name2") + tts_speaker_name3: _Text | None = Field("", alias="_tts_speaker_name3") + tts_speaker_name4: _Text | None = Field("", alias="_tts_speaker_name4") + tts_speaker_name5: _Text | None = Field("", alias="_tts_speaker_name5") + tts_speaker_name6: _Text | None = Field("", alias="_tts_speaker_name6") + tts_voice_count: _VoiceCount = Field(0, alias="_tts_voice_count") + + # Model-owned timing and sampling controls. Duration zero is a real + # native sentinel for DramaBox and therefore remains valid here. + duration_seconds: _NonNegativeFinite | None = None + pause_seconds: _NonNegativeFinite | None = None + temperature: _NonNegativeFinite | None = None + top_p: Annotated[StrictFloat, Field(ge=0, le=1, allow_inf_nan=False)] | None = None + top_k: _NonNegativeInt | None = None + audio_scale: _Finite | None = None + audio_guidance_scale: _Finite | None = None + alt_scale: _Finite | None = None + guidance_phases: Annotated[StrictInt, Field(ge=0, le=16)] = 1 + flow_shift: _Finite | None = None + sample_solver: _ShortText = "" + settings_version: _NonNegativeFinite | None = None + # Speech text must reach TTS literally. Prompt enhancement is a visual + # generation feature and an active value would rewrite the speaker text. + prompt_enhancer: Literal["", None] = "" + model_mode: _ModelMode | None = None + custom_settings: StudioSpeechCustomSettings | None = None + + # Studio's audio UI currently serializes this checkbox as integer 0/1; + # older snapshots may contain a JSON boolean. Both are typed and bounded. + tts_dynaudnorm: _TtsDynaudnorm | None = None + tts_comp_threshold: _Finite | None = None + tts_comp_attack: _NonNegativeFinite | None = None + tts_comp_release: _NonNegativeFinite | None = None + tts_comp_makeup: _NonNegativeFinite | None = None + + # H3's inert sentinel appears in the shared Studio state. A true value is + # a different video/audio operation and must fail closed here. + minimax_h3_turbo_mode: StrictBool | None = None + + @field_validator( + "audio_guide", + "audio_guide2", + "audio_guide3", + "audio_guide4", + "audio_guide5", + "audio_guide6", + ) + @classmethod + def _check_audio_reference(cls, value): + if value in (None, ""): + return value + return _validate_reference(value) + + @field_validator("activated_loras") + @classmethod + def _check_lora_names(cls, values): + for value in values: + if not value.strip() or "/" in value or "\\" in value or value in {".", ".."}: + raise ValueError("activated_loras must contain exact catalog names") + return values + + @field_validator( + "image_start", + "image_end", + "image_guide", + "image_mask", + ) + @classmethod + def _check_inactive_image_slots(cls, value): + if value is None or value == "": + return value + if isinstance(value, list) and all(item == "" for item in value): + return value + raise ValueError("image references are inactive in speech mode") + + @field_validator("image_refs", "h3_ref_videos", "h3_ref_audios", "minimax_h3_references") + @classmethod + def _check_inactive_reference_lists(cls, value): + if value is None or value == [] or (isinstance(value, list) and all(item == "" for item in value)): + return value + raise ValueError("reference lists are inactive in speech mode") + + @field_validator("wangp_processor_settings") + @classmethod + def _check_inactive_processor_settings(cls, value): + if value is None or value == {}: + return value + raise ValueError("processor settings are inactive in speech mode") + + @field_validator("tts_original_prompt") + @classmethod + def _check_original_prompt(cls, value): + if value is None: + # Omission is handled by freeze_studio_speech_spec. Explicit + # JSON null is not a valid native TTS text field. + return value + return value + + @field_validator("minimax_h3_turbo_mode") + @classmethod + def _check_inactive_turbo(cls, value): + if value is True: + raise ValueError("minimax_h3_turbo_mode must be false or null for speech") + return value + + @model_validator(mode="after") + def _check_semantics(self): + _check_required_speech_texts(self) + _check_speech_prompt_metadata(self) + _check_speech_ranges(self) + return self + + +class StudioSpeechInput(_ClosedModel): + workspace: _Workspace + workspace_collection_id: _WorkspaceCollectionId | None = None + params: StudioSpeechParams + + @model_validator(mode="after") + def _check_collection_id(self): + if self.workspace_collection_id is not None and not self.workspace_collection_id.strip(): + raise ValueError("workspace_collection_id must contain a non-blank value") + return self + + +class _StudioSpeechEnvelope(_ClosedModel): + version: Literal[SCHEMA_VERSION] + operation: Literal[OPERATION] + intent_id: _IntentId + input: StudioSpeechInput + + @model_validator(mode="after") + def _check_intent(self): + if not self.intent_id.strip(): + raise ValueError("intent_id must contain a non-blank value") + return self + + +def _validation_error(exc: ValidationError) -> StudioSpeechSpecError: + details: list[dict[str, Any]] = [] + messages: list[str] = [] + for error in exc.errors(include_input=False): + location = ".".join(str(part) for part in error.get("loc", ())) or "command" + message = str(error.get("msg") or "Invalid value") + details.append({"loc": location, "message": message, "type": error.get("type")}) + messages.append(f"{location}: {message}") + return StudioSpeechSpecError( + "; ".join(messages) or "Invalid Studio speech generation command", + details=details, + ) + + +def _canonical_content(effective: dict[str, Any]) -> dict[str, Any]: + return { + "version": SCHEMA_VERSION, + "operation": OPERATION, + "input": deepcopy(effective["input"]), + } + + +def _fingerprint(content: dict[str, Any]) -> str: + encoded = json.dumps( + content, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def freeze_studio_speech_spec(command: Any) -> dict[str, Any]: + """Validate and detach one Studio speech command without side effects.""" + if type(command) is not dict: + raise StudioSpeechSpecError("Studio speech generation command must be an object") + try: + envelope = _StudioSpeechEnvelope.model_validate(command) + except ValidationError as exc: + raise _validation_error(exc) from exc + + original = deepcopy(command) + explicit_params = envelope.input.params.model_dump( + mode="json", by_alias=True, exclude_unset=True + ) + effective_params = deepcopy(explicit_params) + for key, value in STUDIO_SPEECH_DEFAULTS.items(): + effective_params.setdefault(key, deepcopy(value)) + # The original prompt is a native bookkeeping field. When Studio omits + # it, the effective projection gets the literal prompt; an explicit value + # is preserved byte-for-byte in ``original`` and by its own field. + effective_params.setdefault("_tts_original_prompt", explicit_params["prompt"]) + + effective = { + "version": SCHEMA_VERSION, + "operation": OPERATION, + "intent_id": envelope.intent_id, + "input": { + "workspace": envelope.input.workspace, + "params": effective_params, + }, + } + explicit_input = envelope.input.model_dump(mode="json", exclude_unset=True) + if "workspace_collection_id" in explicit_input: + effective["input"]["workspace_collection_id"] = explicit_input["workspace_collection_id"] + content = _canonical_content(effective) + return { + "original": original, + "effective": effective, + "fingerprint_version": FINGERPRINT_VERSION, + "fingerprint": _fingerprint(content), + } + + +def studio_speech_schema() -> dict[str, Any]: + """Return the discovery schema for the implemented speech boundary.""" + input_schema = StudioSpeechInput.model_json_schema() + envelope_schema = _StudioSpeechEnvelope.model_json_schema() + return { + "version": SCHEMA_VERSION, + "operation": OPERATION, + "intent_id": envelope_schema["properties"]["intent_id"], + "input": input_schema, + "supported_input_fields": list(SUPPORTED_INPUT_FIELDS), + "speech_model_types": sorted(SPEECH_MODEL_TYPES), + "effects": deepcopy(STUDIO_SPEECH_DEFAULTS), + "inactive": list(INACTIVE_SPEECH_FIELDS), + "excluded": list(EXCLUDED_SPEECH_FIELDS), + } + + +# Compatibility aliases used by discovery/adapters in the shared command +# slices. They point to this one contract and do not create another schema. +StudioSpeechGenerationInput = StudioSpeechInput +StudioSpeechGenerationParams = StudioSpeechParams +speech_generation_schema_v2 = studio_speech_schema + + +__all__ = [ + "EXCLUDED_SPEECH_FIELDS", + "FINGERPRINT_VERSION", + "INACTIVE_SPEECH_FIELDS", + "OPERATION", + "SCHEMA_VERSION", + "SPEECH_MODEL_TYPES", + "STUDIO_SPEECH_DEFAULTS", + "SUPPORTED_INPUT_FIELDS", + "StudioSpeechGenerationInput", + "StudioSpeechGenerationParams", + "StudioSpeechCustomSettings", + "StudioSpeechInput", + "StudioSpeechParams", + "StudioSpeechSpecError", + "freeze_studio_speech_spec", + "speech_generation_schema_v2", + "studio_speech_schema", +] diff --git a/docs/development/SPEECH_COMMANDS.md b/docs/development/SPEECH_COMMANDS.md new file mode 100644 index 000000000..d0a8f62ea --- /dev/null +++ b/docs/development/SPEECH_COMMANDS.md @@ -0,0 +1,73 @@ +# Studio speech commands + +The shared command boundary for Studio speech is `version=2`, +`operation=generation.speech`: + +```json +{ + "version": 2, + "operation": "generation.speech", + "intent_id": "stable-client-intention", + "input": { + "workspace": "output-workspace", + "params": { + "prompt": "literal speech text", + "model_type": "kugelaudio_0_open", + "resolution": "1280x720", + "num_inference_steps": 0, + "guidance_scale": 3, + "seed": -1, + "generation_mode": "audio", + "image_mode": 0, + "video_length": 0, + "_audio_sub_mode": "speech", + "duration_seconds": 20 + } + } +} +``` + +`freeze_studio_speech_spec` validates and deep-copies the complete native +parameter map. The `original` snapshot keeps submitted spelling, omissions, +Unicode and multiline prompt text. The `effective` snapshot adds only the +speech selectors and other deterministic inactive/default values owned by the +adapter. It does not guess a model mode, duration, provider setting or voice +reference. The fingerprint is SHA-256 over version, operation, workspace and +effective parameters; `intent_id` and caller metadata are excluded so a retry +with another transport identity can be compared by content. + +The accepted speech model registration is deliberately explicit: + +`kugelaudio_0_open`, `qwen3_tts_customvoice`, `qwen3_tts_voicedesign`, +`qwen3_tts_base`, `chatterbox`, `index_tts2`, `scenema_audio`, and +`dramabox_audio`. The preflight requires the selected definition to declare +`audio_only=true`, no image output, and locally downloaded model files. Music +and SFX handlers also advertise audio-only and share the `tts` family, so +their metadata is not sufficient to enter this operation. + +Speech references use exact asset IDs or canonical local API URLs. The +resource adapter checks source-workspace containment, file identity and +positive audio format/duration metadata with `ffprobe` through the existing +`StudioSpeechResources` implementation; this preparation step does not claim +full audio decoding or copy the source. Host paths, traversal, remote URLs and +source-workspace overrides are rejected. + +The private `_tts_*` fields are typed aliases in the JSON contract. Speaker +names remain strings (or an inactive null), `_tts_voice_count` is an integer +from 0 through 6, and `_tts_original_prompt` remains separate from any +native speaker-tag rewrite. `audio_prompt_type` is checked against the +selected handler's declared choices; voice-count-driven handlers must match +their declared count mapping. `duration_seconds=0` is retained when the +selected model declares it as an auto-duration sentinel. + +`custom_settings` is a closed map of the currently registered speech setting +IDs (`auto_split_every_s`, `exaggeration`, `pace`, `vc_steps`, `vc_cfg_rate`, +and `duration_multiplier`). The selected model's own `custom_settings` +metadata supplies the allowed subset and numeric ranges. Native handler +validation remains the final authority for provider-specific prompt syntax, +speaker tags and generative settings. + +This slice intentionally does not announce music, SFX, video, avatar, 3D, +remote providers, arbitrary JSON options or a second scheduler. LoRA use is +accepted only when a speech definition explicitly declares +`enabled_audio_lora`; otherwise selected LoRAs fail before resource lookup. diff --git a/tests/test_studio_speech_runtime_review.py b/tests/test_studio_speech_runtime_review.py new file mode 100644 index 000000000..c92b1f21d --- /dev/null +++ b/tests/test_studio_speech_runtime_review.py @@ -0,0 +1,195 @@ +"""Provider-free checks for the speech adapter's runtime wiring. + +These tests deliberately stop at the native request boundary. They exercise +the real runtime factory and the real speech preparation adapter, while the +``generate`` callable is a small stand-in for the existing native facade. No +model, worker or provider is loaded. +""" + +from __future__ import annotations + +import asyncio +from copy import deepcopy +import json +from pathlib import Path +import threading + +from services.image_generation_runtime import create_image_generation_commands +from services.native_generation_operation import NativeGenerationOperation +from services.task_manager import TaskRegistry +from services.wangp_submission import JsonRequest, prepare_generation_inputs + + +FIXTURE = Path(__file__).parent / "fixtures" / "studio_speech_native_request.json" + + +def _run(awaitable): + return asyncio.run(awaitable) + + +def _speech_command(intent_id="runtime-speech-intent"): + params = json.loads(FIXTURE.read_text(encoding="utf-8")) + workspace = params.pop("workspace") + return { + "version": 2, + "operation": "generation.speech", + "intent_id": intent_id, + "input": {"workspace": workspace, "params": params}, + } + + +def _task_fields(job): + return { + "id": f"task-{job['id']}", + "root_id": f"task-{job['id']}", + "kind": "audio", + "workflow": "generation", + "title": "Audio generation", + "status": "queued", + "phase": "queued", + "message": "Queued", + "workspace": job["workspace"], + "backend_job_id": job["id"], + "current": 0, + "total": 1, + "resource_requirements": ["local_gpu:0"], + "recoverable": True, + } + + +class _Queue: + def __init__(self): + self.jobs = [] + + def upsert(self, job): + self.jobs.append(deepcopy(job)) + + +class _ExecutionMode: + class ExecutionModeError(RuntimeError): + pass + + def validate_generation(self, _workspace): + return None + + +class _WGP: + primary_settings = {} + + def __init__(self, definition): + self.definition = definition + + def get_model_def(self, model_type): + return deepcopy(self.definition) if model_type == "kugelaudio_0_open" else None + + def get_lora_search_dirs(self, _model_type): + return [] + + +def _runtime(tmp_path, observed): + definition = { + "audio_only": True, + "image_outputs": False, + "inference_steps": False, + "guidance_max_phases": 1, + "duration_slider": {"min": 1, "max": 600, "default": 20}, + "audio_prompt_type_sources": {"selection": ["", "A", "AB"], "default": ""}, + "audio_mode_from_voice_count": True, + "max_voice_count": 6, + } + queue = _Queue() + registry = TaskRegistry(str(tmp_path / "runtime-speech"), interrupt_stale=False) + jobs = {} + dispatched = [] + started = threading.Event() + + def make_job(params, workspace, *, job_id=None, created_at=None, + reserve_generation=False, publish_task=False, provenance=None): + del reserve_generation, publish_task + job = { + "id": job_id or "runtime-speech-job", + "status": "queued", + "created_at": created_at or 1000.0, + "params": deepcopy(params), + "workspace": workspace, + "provenance": deepcopy(provenance or {}), + } + return job + + async def native_generate(request): + body = await request.json() + observed["prepared_speech"] = getattr(request, "prepared_studio_speech", False) + observed["trusted_tool"] = getattr(request, "trusted_tool", None) + observed["body_before_native_boundary"] = deepcopy(body) + # This is the same in-process capability handoff performed by the + # real /api/v1/generate route after its ordinary validation. + workspace = body.pop("workspace") + provenance = body.pop("provenance") + prepare_generation_inputs( + body, + definition, + workspace, + uploads_dir=str(tmp_path / "uploads"), + workspace_dir=str(tmp_path / workspace), + prepared_speech=getattr(request, "prepared_studio_speech", False) is True, + ) + observed["body_after_native_preparation"] = deepcopy(body) + return request.admit_generation_command(body, workspace, provenance) + + runtime = { + "_durable_generation_queue": queue, + "_run_generation_with_preparation": lambda _job_id: started.set(), + "_jobs": jobs, + "register_generation_job": lambda _lock, job: dispatched.append(("registered", deepcopy(job))), + "_gen_lock": object(), + "_cancel_h3_idle_release": lambda: None, + "_active_gen_states": {}, + "_task_registry": lambda _workspace: registry, + "generate": native_generate, + "_new_generation_job": make_job, + "_generation_task_fields": _task_fields, + "execution_mode": _ExecutionMode(), + "wgp": _WGP(definition), + "_check_model_downloaded": lambda _model_type: True, + "_workspace_dir": lambda workspace: str(tmp_path / workspace), + "_list_workspaces": lambda: [{"name": "runtime-speech"}], + "_lora_is_compatible_with_model": lambda _definition, _path: True, + } + service = create_image_generation_commands(runtime) + return service, registry, queue, dispatched, started + + +def test_runtime_factory_speech_adapter_reaches_native_boundary_once(tmp_path): + observed = {} + service, registry, queue, dispatched, started = _runtime(tmp_path, observed) + + adapter = service.operations["generation.speech"] + assert isinstance(adapter, NativeGenerationOperation) + assert adapter.catalog["name"] == "generation.speech" + + command = _speech_command() + result = _run(service.submit(command, trusted_tool="external_agent")) + + assert result["replayed"] is False + assert result["receipt"]["operation"] == "generation.speech" + assert observed["prepared_speech"] is True + assert observed["trusted_tool"] == "external_agent" + assert observed["body_before_native_boundary"]["generation_mode"] == "audio" + assert observed["body_after_native_preparation"]["generation_mode"] == "audio" + assert len(dispatched) == 1 + assert started.wait(1) + assert len(queue.jobs) == 1 + + entry = registry.command_admission(command["intent_id"]) + assert entry is not None + assert entry["operation"] == "generation.speech" + assert entry["receipt"] == result["receipt"] + assert entry["effective"]["runtime"]["params"]["prompt"] == command["input"]["params"]["prompt"] + + +def test_json_payload_cannot_forge_prepared_speech_capability(): + request = JsonRequest({"prepared_studio_speech": True}) + + # The marker is an in-process capability set by ImageGenerationCommands; + # arbitrary JSON keys must not become trusted request attributes. + assert getattr(request, "prepared_studio_speech", False) is False diff --git a/tests/test_studio_speech_spec.py b/tests/test_studio_speech_spec.py new file mode 100644 index 000000000..969a343b0 --- /dev/null +++ b/tests/test_studio_speech_spec.py @@ -0,0 +1,246 @@ +"""Provider-free tests for the closed Studio speech command envelope.""" + +from copy import deepcopy +import hashlib +import json +from pathlib import Path + +import pytest + +from services.studio_speech_spec import ( + STUDIO_SPEECH_DEFAULTS, + StudioSpeechSpecError, + freeze_studio_speech_spec, + studio_speech_schema, +) + + +FIXTURE = Path(__file__).parent / "fixtures" / "studio_speech_native_request.json" + + +def speech_command(intent="speech-intent"): + native = json.loads(FIXTURE.read_text()) + workspace = native.pop("workspace") + return { + "version": 2, + "operation": "generation.speech", + "intent_id": intent, + "input": {"workspace": workspace, "params": native}, + } + + +def test_captured_studio_body_round_trips_without_trimming_or_dropping_fields(): + command = speech_command() + before = deepcopy(command) + frozen = freeze_studio_speech_spec(command) + + assert command == before + assert frozen["original"] == before + assert frozen["original"] is not command + assert frozen["original"]["input"]["params"]["prompt"] == "The system is watching.\nEvery warning matters." + assert frozen["effective"]["input"]["params"]["_tts_original_prompt"] == before["input"]["params"]["_tts_original_prompt"] + assert frozen["effective"]["input"]["params"]["duration_seconds"] == 20.0 + assert set(before["input"]["params"]) <= set(frozen["effective"]["input"]["params"]) + + +def test_effective_defaults_are_native_speech_selectors_and_original_omissions_survive(): + command = speech_command() + params = command["input"]["params"] + for key in ( + "generation_mode", + "_audio_sub_mode", + "video_length", + "image_mode", + "multi_prompts_gen_type", + "negative_prompt", + "repeat_generation", + "activated_loras", + "loras_multipliers", + ): + params.pop(key, None) + params.pop("_tts_original_prompt") + frozen = freeze_studio_speech_spec(command) + effective = frozen["effective"]["input"]["params"] + + assert "generation_mode" not in command["input"]["params"] + assert "_tts_original_prompt" not in command["input"]["params"] + for key, value in STUDIO_SPEECH_DEFAULTS.items(): + assert effective[key] == value + assert effective["_tts_original_prompt"] == params["prompt"] + + +@pytest.mark.parametrize("value", ["cinematic", 1, False, [], {}]) +def test_prompt_enhancer_is_rejected_when_active_or_wrongly_typed(value): + command = speech_command() + command["input"]["params"]["prompt_enhancer"] = value + + with pytest.raises(StudioSpeechSpecError): + freeze_studio_speech_spec(command) + + +@pytest.mark.parametrize("value", ["", None]) +def test_prompt_enhancer_inactive_sentinels_are_preserved(value): + command = speech_command() + command["input"]["params"]["prompt_enhancer"] = value + + frozen = freeze_studio_speech_spec(command) + + assert frozen["effective"]["input"]["params"]["prompt_enhancer"] == value + + +def test_fingerprint_excludes_transport_intent_but_covers_workspace_and_native_content(): + first = freeze_studio_speech_spec(speech_command("one")) + second = freeze_studio_speech_spec(speech_command("two")) + assert first["fingerprint"] == second["fingerprint"] + changed = speech_command("three") + changed["input"]["params"]["prompt"] += " changed" + assert freeze_studio_speech_spec(changed)["fingerprint"] != first["fingerprint"] + other_workspace = speech_command("four") + other_workspace["input"]["workspace"] = "another-workspace" + assert freeze_studio_speech_spec(other_workspace)["fingerprint"] != first["fingerprint"] + + +def test_fingerprint_is_stable_and_uses_canonical_json(): + frozen = freeze_studio_speech_spec(speech_command()) + content = { + "version": 2, + "operation": "generation.speech", + "input": frozen["effective"]["input"], + } + expected = hashlib.sha256( + json.dumps(content, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + ).hexdigest() + assert frozen["fingerprint"] == expected + assert len(frozen["fingerprint"]) == 64 + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("generation_mode", "image"), + ("_audio_sub_mode", "music"), + ("image_mode", 1), + ("video_length", 1), + ("multi_prompts_gen_type", 1), + ("minimax_h3_turbo_mode", True), + ("audio_prompt_type", "A/host-path"), + ], +) +def test_speech_mode_rejects_video_music_and_active_h3_values(field, value): + command = speech_command() + command["input"]["params"][field] = value + with pytest.raises(StudioSpeechSpecError): + freeze_studio_speech_spec(command) + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("num_inference_steps", True), + ("num_inference_steps", 1.0), + ("_tts_voice_count", "1"), + ("_tts_voice_count", True), + ("duration_seconds", "20"), + ("duration_seconds", float("nan")), + ("temperature", True), + ("top_k", 2.5), + ("tts_dynaudnorm", 2), + ], +) +def test_native_scalars_are_strict_and_finite(field, value): + command = speech_command() + command["input"]["params"][field] = value + with pytest.raises(StudioSpeechSpecError): + freeze_studio_speech_spec(command) + + +@pytest.mark.parametrize( + "value", + [ + "/etc/passwd", + "../voice.wav", + "C:\\voice.wav", + "https://example.test/voice.wav", + "/api/v1/file/voice.wav", + "/api/v1/file/../voice.wav?workspace=speech-test", + "/api/v1/uploads/voice.wav?workspace=other", + "asset_abc?workspace=other", + ], +) +def test_audio_references_are_canonical_and_workspace_explicit(value): + command = speech_command() + command["input"]["params"].update( + {"audio_prompt_type": "A", "_tts_voice_count": 1, "audio_guide": value} + ) + with pytest.raises(StudioSpeechSpecError): + freeze_studio_speech_spec(command) + + +@pytest.mark.parametrize( + "value", + [ + "", + None, + "asset_abc123", + "/api/v1/uploads/voice.wav", + "/api/v1/file/voice.wav?workspace=speech-test", + "/api/v1/assets/asset_abc123", + ], +) +def test_audio_reference_sentinels_and_canonical_urls_are_retained(value): + command = speech_command() + command["input"]["params"].update( + {"audio_prompt_type": "", "_tts_voice_count": 0, "audio_guide": value} + ) + frozen = freeze_studio_speech_spec(command) + assert frozen["original"]["input"]["params"]["audio_guide"] == value + assert frozen["effective"]["input"]["params"]["audio_guide"] == value + + +def test_inactive_shared_state_is_retained_only_as_empty_sentinels(): + command = speech_command() + params = command["input"]["params"] + params.update( + { + "image_start": ["", ""], + "image_refs": [], + "h3_ref_videos": [], + "spatial_upsampling": "", + "wangp_processor_settings": {}, + "MMAudio_setting": 0, + "MMAudio_prompt": None, + } + ) + frozen = freeze_studio_speech_spec(command) + assert frozen["effective"]["input"]["params"]["image_start"] == ["", ""] + assert frozen["effective"]["input"]["params"]["MMAudio_setting"] == 0 + params["image_start"] = ["/api/v1/uploads/image.png"] + with pytest.raises(StudioSpeechSpecError): + freeze_studio_speech_spec(command) + + +def test_custom_settings_are_closed_and_authority_fields_are_rejected(): + command = speech_command() + command["input"]["params"]["custom_settings"] = {"auto_split_every_s": 5} + frozen = freeze_studio_speech_spec(command) + assert frozen["effective"]["input"]["params"]["custom_settings"] == {"auto_split_every_s": 5.0} + for key, value in (("unknown", 1), ("provenance", {}), ("actor", "user")): + invalid = speech_command() + if key in {"provenance", "actor"}: + invalid["input"]["params"][key] = value + else: + invalid["input"]["params"]["custom_settings"] = {key: value} + with pytest.raises(StudioSpeechSpecError): + freeze_studio_speech_spec(invalid) + + +def test_schema_exposes_closed_speech_boundary_and_supported_models(): + schema = studio_speech_schema() + assert schema["version"] == 2 + assert schema["operation"] == "generation.speech" + params_schema = schema["input"]["$defs"]["StudioSpeechParams"] + assert params_schema["additionalProperties"] is False + assert "_tts_original_prompt" in params_schema["properties"] + assert "kugelaudio_0_open" in schema["speech_model_types"] + assert "minimax_music3" not in schema["speech_model_types"] + assert "provenance" in schema["excluded"] From 11e26a0fb3f8696c7b070bad5431fc8d613737b9 Mon Sep 17 00:00:00 2001 From: IAnMove <216241348+IAnMove@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:25:01 +0200 Subject: [PATCH 02/59] feat(studio): share durable speech commands across UI Wizard and MCP --- app/_launch_runtime.py | 6 +- app/routers/image_generation_commands.py | 27 +- app/routers/studio_speech_commands.py | 20 + app/services/image_generation_commands.py | 38 +- app/services/image_generation_runtime.py | 27 +- app/services/native_generation_operation.py | 14 + app/services/studio_image_resources.py | 6 +- app/services/studio_speech_resources.py | 35 + app/services/wangp_submission.py | 21 +- scripts/export_speech_command_catalog.py | 28 + .../studio_speech_native_request.json | 29 + tests/test_studio_speech_commands.py | 134 ++ tests/test_studio_speech_preparation.py | 349 +++ tests/test_studio_speech_resources.py | 112 + ui/src/api/generationCommandClient.ts | 719 ++++++ ui/src/api/imageCommandCatalog.json | 2 +- ui/src/api/imageGenerationCommands.ts | 769 +------ ui/src/api/speechCommandCatalog.json | 1973 +++++++++++++++++ ui/src/api/speechGenerationCommands.ts | 92 + ui/src/components/Sidebar/Sidebar.tsx | 23 +- ui/src/features/agent/studioCapabilities.ts | 23 +- .../features/studio/StudioCommandPanels.tsx | 30 + .../studio/StudioSpeechCommandPanel.tsx | 173 ++ ui/src/features/studio/actions.ts | 4 +- .../studio/speechCommandPresentation.ts | 77 + .../studio/speechCommandSubmission.ts | 178 ++ .../features/studio/speechGenerationSpec.ts | 530 +++++ ui/src/features/studio/studioSubmission.ts | 29 +- ui/src/i18n/locales/en/studio.json | 12 + ui/src/i18n/locales/es/studio.json | 12 + ui/src/stores/useStore.ts | 4 +- ui/tests/studioCapabilities.test.mjs | 38 + .../studioSpeechCommandPresentation.test.tsx | 316 +++ .../studioSpeechGenerationCommands.test.ts | 235 ++ 34 files changed, 5297 insertions(+), 788 deletions(-) create mode 100644 app/routers/studio_speech_commands.py create mode 100644 app/services/native_generation_operation.py create mode 100644 app/services/studio_speech_resources.py create mode 100644 scripts/export_speech_command_catalog.py create mode 100644 tests/fixtures/studio_speech_native_request.json create mode 100644 tests/test_studio_speech_commands.py create mode 100644 tests/test_studio_speech_preparation.py create mode 100644 tests/test_studio_speech_resources.py create mode 100644 ui/src/api/generationCommandClient.ts create mode 100644 ui/src/api/speechCommandCatalog.json create mode 100644 ui/src/api/speechGenerationCommands.ts create mode 100644 ui/src/features/studio/StudioCommandPanels.tsx create mode 100644 ui/src/features/studio/StudioSpeechCommandPanel.tsx create mode 100644 ui/src/features/studio/speechCommandPresentation.ts create mode 100644 ui/src/features/studio/speechCommandSubmission.ts create mode 100644 ui/src/features/studio/speechGenerationSpec.ts create mode 100644 ui/tests/studioSpeechCommandPresentation.test.tsx create mode 100644 ui/tests/studioSpeechGenerationCommands.test.ts diff --git a/app/_launch_runtime.py b/app/_launch_runtime.py index 81254cee0..26f874029 100644 --- a/app/_launch_runtime.py +++ b/app/_launch_runtime.py @@ -10870,7 +10870,8 @@ async def generate(request: Request): prepare_generation_inputs(body, _generation_model_def, requested_workspace, uploads_dir=os.path.join(os.getcwd(), "uploads"), workspace_dir=_workspace_dir(requested_workspace), - prepared_images=getattr(request, "prepared_studio_images", False) is True) + prepared_images=getattr(request, "prepared_studio_images", False) is True, + prepared_speech=getattr(request, "prepared_studio_speech", False) is True) except ValueError as error: raise HTTPException(status_code=400, detail=str(error)) from error try: @@ -36863,7 +36864,8 @@ def _classic_redirect(): "generate": generate, "recast": recast_endpoint, "upscale": tools_upscale, **wangp_agent_handlers(api), **image_command_handlers(_image_generation_commands)}, journal_path=os.path.join(os.path.dirname(__file__), "settings", "wangp-mcp-requests.sqlite3"), - command_operations=[*workspace_command_catalog()["operations"], *image_command_catalog()], + command_operations=[*workspace_command_catalog()["operations"], *image_command_catalog( + adapter.catalog for adapter in _image_generation_commands.operations.values())], )) # ============================================================================ diff --git a/app/routers/image_generation_commands.py b/app/routers/image_generation_commands.py index 9b32269f0..05d6353ac 100644 --- a/app/routers/image_generation_commands.py +++ b/app/routers/image_generation_commands.py @@ -1,5 +1,6 @@ """HTTP and MCP projections of the executable image command contract.""" from __future__ import annotations +from typing import Literal from fastapi import APIRouter, Request from pydantic import BaseModel, ConfigDict, Field, StrictStr, ValidationError, field_validator @@ -11,6 +12,7 @@ class ReferenceResolutionInput(BaseModel): model_config = ConfigDict(extra="forbid", strict=True) references: list[StrictStr] = Field(min_length=1, max_length=64) + media_kind: Literal["image", "audio"] = "image" class UISubmissionContext(BaseModel): @@ -37,7 +39,7 @@ def _ui_context(request): raise command_error(422, "invalid_ui_context", "Use only exact workflowId and runId attribution") from error -def image_command_catalog(): +def image_command_catalog(additional_operations=()): spec = image_generation_schema() studio = studio_image_schema() studio_input = dict(studio["input"]) @@ -60,18 +62,20 @@ def image_command_catalog(): "description": "Admit an image job with an installed model and explicit output workspace. Version 1 is a single text-to-image request; version 2 accepts the complete typed Studio image parameters, canonical references, LoRAs and image processors. Preserve literal prompts and reuse intent_id only for retries. The receipt proves admission; inspect its task for completion.", "inputSchema": envelope}, {"name": "generation.receipt", "version": 1, "domain": "studio", "mutation": False, - "description": "Read an immutable image admission and its current canonical task in the exact original output workspace.", + "description": "Read an immutable generation admission and its current canonical task in the exact original output workspace.", "inputSchema": {"type": "object", "additionalProperties": False, "properties": {"version": {"type": "integer", "const": 1}, "operation": {"const": "generation.receipt"}, "input": receipt_input}, - "required": ["version", "operation", "input"]}}] + "required": ["version", "operation", "input"]}}, *additional_operations] def image_command_handlers(service): - async def submit(arguments): - if not isinstance(arguments, dict) or set(arguments) != {"version", "intent_id", "input"}: - raise command_error(422, "invalid_command", "Use version, intent_id and input for the image tool") - return await service.submit({**arguments, "operation": "generation.image"}, trusted_tool="external_agent") + def submission_handler(operation): + async def submit(arguments): + if not isinstance(arguments, dict) or set(arguments) != {"version", "intent_id", "input"}: + raise command_error(422, "invalid_command", "Use version, intent_id and input for the generation tool") + return await service.submit({**arguments, "operation": operation}, trusted_tool="external_agent") + return submit def receipt(arguments): if (not isinstance(arguments, dict) or set(arguments) != {"version", "input"} @@ -81,7 +85,8 @@ def receipt(arguments): raise command_error(422, "invalid_command", "Use version 1 with workspace and intent_id") return service.receipt(**arguments["input"]) - return {"generation.image": submit, "generation.receipt": receipt} + operations = {"generation.image", *getattr(service, "operations", {})} + return {**{operation: submission_handler(operation) for operation in operations}, "generation.receipt": receipt} def create_image_generation_commands_router(service): @@ -89,7 +94,8 @@ def create_image_generation_commands_router(service): @router.get("/api/v1/generation/commands") def catalog(): - return {"version": 2, "operations": image_command_catalog()} + return {"version": 2, "operations": image_command_catalog( + adapter.catalog for adapter in getattr(service, "operations", {}).values())} @router.post("/api/v1/generation/commands") async def submit(request: Request): @@ -118,7 +124,8 @@ def references(body: ReferenceResolutionInput): if any(not 1 <= len(value) <= 8192 for value in body.references): raise command_error(422, "invalid_reference", "An exact bounded media reference is required") try: - return {"references": [resolve(value) for value in body.references]} + return {"references": [resolve(value) if body.media_kind == "image" + else resolve(value, media_kind="audio") for value in body.references]} except (ValueError, OSError) as error: raise command_error(422, "invalid_reference", str(error)) from error diff --git a/app/routers/studio_speech_commands.py b/app/routers/studio_speech_commands.py new file mode 100644 index 000000000..22f08ca47 --- /dev/null +++ b/app/routers/studio_speech_commands.py @@ -0,0 +1,20 @@ +"""Serializable speech operation projected through the shared HTTP/MCP router.""" +from services.studio_speech_spec import studio_speech_schema + + +def speech_command_catalog(): + schema = studio_speech_schema() + input_schema = dict(schema["input"]) + definitions = input_schema.pop("$defs", {}) + return { + "name": "generation.speech", "version": 2, "supportedVersions": [2], + "domain": "studio", "mutation": True, + "description": "Admit speech using an installed speech model, literal text, voice settings and canonical audio references in an explicit output workspace. Preserve the original speaker text separately from the effective native prompt. Reuse intent_id only for retries; the receipt proves admission, and its task reports completion.", + "inputSchema": { + "type": "object", "additionalProperties": False, "$defs": definitions, + "properties": {"version": {"type": "integer", "const": 2}, + "operation": {"const": "generation.speech"}, + "intent_id": schema["intent_id"], "input": input_schema}, + "required": ["version", "operation", "intent_id", "input"], + }, + } diff --git a/app/services/image_generation_commands.py b/app/services/image_generation_commands.py index 1a66945c8..23c79fba3 100644 --- a/app/services/image_generation_commands.py +++ b/app/services/image_generation_commands.py @@ -1,4 +1,4 @@ -"""Shared image admission using native preparation, tasks and generation FIFO. +"""Shared native admission using preparation, tasks and the generation FIFO. The receipt proves admission. TaskRegistry remains the progress authority and the native generation queue remains the sole execution/recovery mechanism. @@ -37,7 +37,8 @@ def validate_image_model(params, *, model_definition, model_downloaded, allow_re class ImageGenerationCommands: def __init__(self, *, registry, prepare, preflight, make_job, task_fields, - dispatch, persist_recovery, active_job_ids, prepare_studio=None, runtime_defaults=None): + dispatch, persist_recovery, active_job_ids, prepare_studio=None, runtime_defaults=None, + operations=None): self.registry = registry self.prepare = prepare self.preflight = preflight @@ -48,6 +49,9 @@ def __init__(self, *, registry, prepare, preflight, make_job, task_fields, self.active_job_ids = active_job_ids self.prepare_studio = prepare_studio self.runtime_defaults = runtime_defaults or (lambda: {}) + self.operations = dict(operations or {}) + if "generation.image" in self.operations: + raise ValueError("The existing image contract cannot be overridden") self.owner = uuid.uuid4().hex def _registry(self, workspace): @@ -59,7 +63,7 @@ def _registry(self, workspace): @staticmethod def _validate_replay(entry, frozen): - if (entry["operation"] != "generation.image" or entry["digest"] != frozen["fingerprint"] + if (entry["operation"] != frozen["original"]["operation"] or entry["digest"] != frozen["fingerprint"] or entry["fingerprint_version"] != frozen["fingerprint_version"]): raise TaskCommandConflict("intent_id was already used with different parameters or preconditions") @@ -101,7 +105,7 @@ def _admit(self, frozen, body, workspace, provenance): effective["runtime"] = {"params": deepcopy(job["params"]), "workspace": workspace, "provenance": deepcopy(job["provenance"])} admitted = registry.admit_command_task( - intent_id=frozen["original"]["intent_id"], operation="generation.image", + intent_id=frozen["original"]["intent_id"], operation=frozen["original"]["operation"], digest=frozen["fingerprint"], original=frozen["original"], effective=effective, task_fields=self.task_fields(job), fingerprint_version=frozen["fingerprint_version"], ) @@ -116,7 +120,7 @@ def _provenance(frozen, trusted_tool, context): if context and context.get(source): command[target] = context[source] result = {"actor": "wizard" if trusted_tool == "wizard" else "user", - "capability": "generation.image", "command": command} + "capability": frozen["original"]["operation"], "command": command} collection = frozen["original"]["input"].get("workspace_collection_id") if collection is not None: result["workspace_id"] = collection @@ -131,7 +135,11 @@ async def submit(self, command, *, trusted_tool=None, submission_context=None): self._validate_replay(previous, frozen) self._dispatch_admitted(registry, previous) return {"receipt": previous["receipt"], "replayed": True} - if command["version"] == 2: + adapter = self.operations.get(command["operation"]) + if adapter is not None: + params, resources = adapter.prepare(params) + frozen["effective"]["resources"] = resources + elif command["version"] == 2: if self.prepare_studio is None: raise command_error(422, "unsupported_version", "Studio image commands are unavailable in this runtime") params, resources = self.prepare_studio(params) @@ -140,7 +148,8 @@ async def submit(self, command, *, trusted_tool=None, submission_context=None): self.preflight(params) request = JsonRequest({**deepcopy(params), "provenance": self._provenance( frozen, trusted_tool, submission_context)}, trusted_tool=trusted_tool) - request.prepared_studio_images = command["version"] == 2 + request.prepared_studio_images = command["operation"] == "generation.image" and command["version"] == 2 + request.prepared_studio_speech = command["operation"] == "generation.speech" # This callback is an in-process capability, never a JSON option. # The native facade performs its ordinary validation first and then # transfers admission to the same canonical task/worker adapter. @@ -153,8 +162,11 @@ async def submit(self, command, *, trusted_tool=None, submission_context=None): except (OSError, sqlite3.Error) as error: raise command_error(503, "storage_unavailable", "Command storage is unavailable; retry with the same intention") from error - @staticmethod - def _freeze(command): + def _freeze(self, command): + if isinstance(command, dict) and isinstance(command.get("operation"), str): + adapter = self.operations.get(command["operation"]) + if adapter is not None: + return adapter.freeze(command) if isinstance(command, dict) and type(command.get("version")) is int and command["version"] == 2: from services.studio_image_spec import freeze_studio_image_spec frozen = freeze_studio_image_spec(command) @@ -183,6 +195,10 @@ def restore_recovery(self, workspaces): for workspace in workspaces: registry = self._registry(workspace) for entry in registry.command_recovery_candidates(): + if entry["operation"] not in {"generation.image", *self.operations}: + # Another domain can share TaskRegistry without using + # this runtime's native generation recovery projection. + continue task = registry.get(entry["task_id"]) if not task or task["status"] != "interrupted" or task["backend_job_id"] in active: continue @@ -194,13 +210,13 @@ def restore_recovery(self, workspaces): def _recovery_task(self, record): provenance = record.get("provenance") or {} - if provenance.get("capability") != "generation.image": + if provenance.get("capability") not in {"generation.image", *self.operations}: return None registry = self._registry(record["workspace"]) intent_id = provenance.get("command", {}).get("command_id") entry = registry.command_admission(intent_id) if entry is None or entry["receipt"]["result"]["job_id"] != record["id"]: - raise command_error(503, "recovery_mismatch", "Recovery does not match a durable image admission") + raise command_error(503, "recovery_mismatch", "Recovery does not match a durable generation admission") return registry, registry.get(entry["task_id"]) def filter_recovery(self, records): diff --git a/app/services/image_generation_runtime.py b/app/services/image_generation_runtime.py index a1e762221..eff3af7c2 100644 --- a/app/services/image_generation_runtime.py +++ b/app/services/image_generation_runtime.py @@ -45,9 +45,11 @@ def preflight(params): validate_image_model(params, model_definition=runtime["wgp"].get_model_def, model_downloaded=runtime["_check_model_downloaded"]) - def resources(): + def resources(media_kind="image"): from services.studio_image_resources import StudioImageResources - return StudioImageResources( + from services.studio_speech_resources import StudioSpeechResources + resource_type = StudioSpeechResources if media_kind == "audio" else StudioImageResources + return resource_type( workspace_dir=runtime["_workspace_dir"], uploads_dir=lambda: os.path.join(os.getcwd(), "uploads"), list_workspaces=runtime["_list_workspaces"], lora_search_dirs=runtime["wgp"].get_lora_search_dirs, lora_compatible=runtime["_lora_is_compatible_with_model"], @@ -63,11 +65,30 @@ def prepare_studio(params): validate_processors=processors.validate_selection, processor_settings=processors.validated_settings, ) + def speech_operation(): + from services.native_generation_operation import NativeGenerationOperation + from services.studio_speech_spec import freeze_studio_speech_spec + from services.studio_speech_preparation import prepare_studio_speech + from routers.studio_speech_commands import speech_command_catalog + + def freeze(command): + frozen = freeze_studio_speech_spec(command) + effective = frozen["effective"]["input"] + return frozen, {**deepcopy(effective["params"]), "workspace": effective["workspace"]} + + def prepare(params): + return prepare_studio_speech(params, model_definition=runtime["wgp"].get_model_def, + model_downloaded=runtime["_check_model_downloaded"], + resources=resources("audio"), execution_policy=execution_policy) + + return NativeGenerationOperation(freeze=freeze, prepare=prepare, catalog=speech_command_catalog()) + service = ImageGenerationCommands( registry=runtime["_task_registry"], prepare=runtime["generate"], preflight=preflight, make_job=runtime["_new_generation_job"], task_fields=runtime["_generation_task_fields"], dispatch=dispatch, persist_recovery=persist, active_job_ids=lambda: runtime["_jobs"].keys(), prepare_studio=prepare_studio, runtime_defaults=lambda: {"mode": "", **runtime["wgp"].primary_settings}, + operations={"generation.speech": speech_operation()}, ) - service.canonicalize_reference = lambda value: resources().canonicalize_legacy(value) + service.canonicalize_reference = lambda value, media_kind="image": resources(media_kind).canonicalize_legacy(value) return service diff --git a/app/services/native_generation_operation.py b/app/services/native_generation_operation.py new file mode 100644 index 000000000..a874f3901 --- /dev/null +++ b/app/services/native_generation_operation.py @@ -0,0 +1,14 @@ +"""Typed adapters for operations admitted into the existing generation queue. + +An adapter freezes caller input without I/O and validates/resolves resources +before native admission. It neither owns task storage nor starts workers. +""" +from dataclasses import dataclass +from typing import Callable + + +@dataclass(frozen=True) +class NativeGenerationOperation: + freeze: Callable[[dict], tuple[dict, dict]] + prepare: Callable[[dict], tuple[dict, list]] + catalog: dict diff --git a/app/services/studio_image_resources.py b/app/services/studio_image_resources.py index a1c1d0b6a..370582612 100644 --- a/app/services/studio_image_resources.py +++ b/app/services/studio_image_resources.py @@ -32,6 +32,8 @@ def file_identity(path): class StudioImageResources: + media_kind = "image" + def __init__(self, *, workspace_dir, uploads_dir, list_workspaces, lora_search_dirs, lora_compatible): self.workspace_dir = workspace_dir @@ -91,8 +93,8 @@ def _asset_url(self, identity): roots = [{"workspace_id": name, "path": self.workspace_dir(name)} for name in self._workspace_names()] roots.append({"workspace_id": "__uploads__", "path": self.uploads_dir()}) asset = find_asset(roots, identity) - if not asset or asset.get("kind") != "image": - raise ValueError("Choose an existing image asset ID") + if not asset or asset.get("kind") != self.media_kind: + raise ValueError(f"Choose an existing {self.media_kind} asset ID") locations = asset.get("locations") or [] if len(locations) != 1: raise ValueError("This asset has multiple locations; choose an exact source URL") diff --git a/app/services/studio_speech_resources.py b/app/services/studio_speech_resources.py new file mode 100644 index 000000000..7f2ab2d80 --- /dev/null +++ b/app/services/studio_speech_resources.py @@ -0,0 +1,35 @@ +"""Inspect canonical speech references without changing their source workspace.""" +from copy import deepcopy +import math +import subprocess + +from services.studio_image_resources import StudioImageResources, file_identity +from services.video_editor import probe_audio + + +SPEECH_AUDIO_FIELDS = ("audio_guide", *(f"audio_guide{i}" for i in range(2, 7))) + + +class StudioSpeechResources(StudioImageResources): + media_kind = "audio" + + def prepare_media(self, params): + working = deepcopy(params) + resources = [] + for field in SPEECH_AUDIO_FIELDS: + value = working.get(field) + if not value: + continue + path, workspace = self._media(value) + identity = file_identity(path) + try: + information = probe_audio(path) + except subprocess.TimeoutExpired as error: + raise ValueError("A selected voice reference could not be inspected in time") from error + duration = information["duration"] + if not math.isfinite(duration) or duration <= 0: + raise ValueError("A selected voice reference has no finite positive duration") + resources.append({"role": field, "index": 0, "url": value, + "workspace": workspace, "duration_seconds": duration, **identity}) + working[field] = path + return working, resources diff --git a/app/services/wangp_submission.py b/app/services/wangp_submission.py index 895dae49a..eea5d3c00 100644 --- a/app/services/wangp_submission.py +++ b/app/services/wangp_submission.py @@ -20,8 +20,7 @@ async def json(self): return deepcopy(self.payload) -def prepare_generation_inputs(body, model_def, workspace, *, uploads_dir, workspace_dir, prepared_images=False): - """Validate processor options and resolve new-family media before admission.""" +def _prepare_canonical_image_references(body, model_def, workspace, *, uploads_dir, workspace_dir): canonical_refs = body.pop('canonical_image_refs', False) if canonical_refs: if canonical_refs is not True or not model_def.get('image_outputs') or body.get('image_mode') != 1: @@ -31,23 +30,39 @@ def prepare_generation_inputs(body, model_def, workspace, *, uploads_dir, worksp raise ValueError('Canonical image references must be a non-empty ordered list') body['image_refs'] = [resolve_wangp_media(value, workspace, uploads_dir=uploads_dir, workspace_dir=workspace_dir) for value in references] + + +def _prepare_native_processors(body): if body.get('spatial_upsampling') or body.get('temporal_upsampling') or body.get('wangp_processor_settings'): from shared.wangp1272.processors import validate_selection, validated_settings error = validate_selection(body.get('spatial_upsampling', ''), body.get('temporal_upsampling', ''), body.get('image_mode') == 1) if error: raise ValueError(error) body['wangp_processor_settings'] = validated_settings(body.get('spatial_upsampling', ''), body.get('wangp_processor_settings')) + + +def prepare_generation_inputs(body, model_def, workspace, *, uploads_dir, workspace_dir, + prepared_images=False, prepared_speech=False): + """Validate processor options and resolve new-family media before admission.""" + _prepare_canonical_image_references(body, model_def, workspace, + uploads_dir=uploads_dir, workspace_dir=workspace_dir) + _prepare_native_processors(body) if not model_def.get('wangp_1272'): return if prepared_images and (not model_def.get('image_outputs') or body.get('image_mode') != 1): raise ValueError('Prepared image inputs require an image generation request') + if prepared_speech and (not model_def.get('audio_only') or body.get('generation_mode') != 'audio'): + raise ValueError('Prepared speech inputs require an audio generation request') def resolve(value): return resolve_wangp_media(value, workspace, uploads_dir=uploads_dir, workspace_dir=workspace_dir) - for field in ('video_guide', 'video_guide2', 'video_mask', 'audio_guide', 'audio_guide2', 'image_start', 'image_end', 'image_refs'): + audio_fields = ('audio_guide', 'audio_guide2', 'audio_guide3', 'audio_guide4', 'audio_guide5', 'audio_guide6') + for field in ('video_guide', 'video_guide2', 'video_mask', *audio_fields, 'image_start', 'image_end', 'image_refs'): if prepared_images and field in ('image_start', 'image_end', 'image_refs'): # Only the in-process Studio command adapter supplies this flag. # Those exact paths were resolved against each source workspace. continue + if prepared_speech and field in audio_fields: + continue values = body.get(field) if values: body[field] = [resolve(value) for value in values] if isinstance(values, list) else resolve(values) diff --git a/scripts/export_speech_command_catalog.py b/scripts/export_speech_command_catalog.py new file mode 100644 index 000000000..0c401f1b8 --- /dev/null +++ b/scripts/export_speech_command_catalog.py @@ -0,0 +1,28 @@ +"""Export the executable Studio speech contract for its browser projection.""" +from pathlib import Path +import argparse +import json +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "app")) +from routers.studio_speech_commands import speech_command_catalog +from services.studio_speech_spec import studio_speech_schema + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + value = {"version": 2, "operations": [speech_command_catalog()], "studio": studio_speech_schema()} + expected = json.dumps(value, indent=2, ensure_ascii=False) + "\n" + target = ROOT / "ui/src/api/speechCommandCatalog.json" + if args.check: + if not target.is_file() or target.read_text(encoding="utf-8") != expected: + raise SystemExit("Speech command projection is stale; regenerate and review its diff") + else: + target.write_text(expected, encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/tests/fixtures/studio_speech_native_request.json b/tests/fixtures/studio_speech_native_request.json new file mode 100644 index 000000000..518e98ec8 --- /dev/null +++ b/tests/fixtures/studio_speech_native_request.json @@ -0,0 +1,29 @@ +{ + "prompt": "The system is watching.\nEvery warning matters.", + "model_type": "kugelaudio_0_open", + "resolution": "1280x720", + "video_length": 0, + "num_inference_steps": 0, + "guidance_scale": 3, + "seed": -1, + "image_mode": 0, + "negative_prompt": "", + "repeat_generation": 1, + "activated_loras": [], + "loras_multipliers": "", + "settings_version": 2.52, + "flow_shift": 5, + "temperature": 1, + "audio_prompt_type": "", + "guidance_phases": 1, + "minimax_h3_turbo_mode": false, + "generation_mode": "audio", + "workspace": "speech-test", + "_audio_sub_mode": "speech", + "multi_prompts_gen_type": 2, + "_tts_original_prompt": "The system is watching.\nEvery warning matters.", + "_tts_speaker_name1": "", + "_tts_speaker_name2": "", + "_tts_voice_count": 0, + "duration_seconds": 20 +} diff --git a/tests/test_studio_speech_commands.py b/tests/test_studio_speech_commands.py new file mode 100644 index 000000000..35fba1b58 --- /dev/null +++ b/tests/test_studio_speech_commands.py @@ -0,0 +1,134 @@ +"""Shared speech admission, native snapshots and inter-client replay without a provider.""" +from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy +import json +from pathlib import Path + +import pytest +from fastapi import FastAPI, HTTPException +from fastapi.testclient import TestClient + +from routers.image_generation_commands import create_image_generation_commands_router, image_command_handlers +from routers.studio_speech_commands import speech_command_catalog +from services.native_generation_operation import NativeGenerationOperation +from services.studio_speech_spec import freeze_studio_speech_spec, studio_speech_schema +from tests.test_image_generation_commands import FakeNative, _command as image_command, _run, _db_counts + + +def speech_command(intent="speech-test-intent"): + native = json.loads((Path(__file__).parent / "fixtures/studio_speech_native_request.json").read_text()) + workspace = native.pop("workspace") + return {"version": 2, "operation": "generation.speech", "intent_id": intent, + "input": {"workspace": workspace, "params": native}} + + +def configured_service(native): + def freeze(command): + frozen = freeze_studio_speech_spec(command) + effective = frozen["effective"]["input"] + return frozen, {**deepcopy(effective["params"]), "workspace": effective["workspace"]} + + def prepare(params): + native.preflight(params) + return deepcopy(params), [] + + service = native.service() + service.operations = {"generation.speech": NativeGenerationOperation( + freeze=freeze, prepare=prepare, catalog=speech_command_catalog())} + return service + + +def test_http_then_mcp_replays_same_speech_task_and_native_literal_snapshot(tmp_path): + native = FakeNative(tmp_path) + service = configured_service(native) + app = FastAPI() + app.include_router(create_image_generation_commands_router(service)) + command = speech_command() + with TestClient(app) as client: + first = client.post("/api/v1/generation/commands", json=command, + headers={"X-Hocus-UI-Surface": "wizard"}) + assert first.status_code == 200 + catalog = client.get("/api/v1/generation/commands").json() + names = {operation["name"] for operation in catalog["operations"]} + assert {"generation.image", "generation.speech", "generation.receipt"} <= names + mcp_arguments = {key: value for key, value in command.items() if key != "operation"} + replay = _run(image_command_handlers(service)["generation.speech"](mcp_arguments)) + receipt = first.json()["receipt"] + assert replay == {"receipt": receipt, "replayed": True} + assert receipt["operation"] == "generation.speech" + assert native.prepare_calls == native.preflight_calls == 1 + assert len(native.dispatch_calls) == 1 + entry = native.registry("speech-test").command_admission(command["intent_id"]) + snapshot = entry["effective"]["runtime"] + assert snapshot["params"]["prompt"] == command["input"]["params"]["prompt"] + assert snapshot["params"]["_tts_original_prompt"] == command["input"]["params"]["_tts_original_prompt"] + assert snapshot["params"]["duration_seconds"] == 20 + assert snapshot["provenance"]["capability"] == "generation.speech" + assert snapshot["provenance"]["actor"] == "wizard" + + +def test_intent_cannot_cross_image_and_speech_domains(tmp_path): + native = FakeNative(tmp_path) + service = configured_service(native) + _run(service.submit(speech_command())) + with pytest.raises(HTTPException) as caught: + _run(service.submit(image_command("speech-test-intent", workspace="speech-test"))) + assert caught.value.status_code == 409 + assert len(native.dispatch_calls) == 1 + + +def test_concurrent_speech_clients_and_deliberate_second_attempt(tmp_path): + native = FakeNative(tmp_path) + service = configured_service(native) + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(lambda _: _run(service.submit(speech_command())), range(2))) + assert results[0]["receipt"] == results[1]["receipt"] + assert sum(not result["replayed"] for result in results) == 1 + assert len(native.dispatch_calls) == 1 + second = _run(service.submit(speech_command("another-deliberate-speech"))) + assert second["receipt"]["taskIds"] != results[0]["receipt"]["taskIds"] + assert _db_counts(native.registry("speech-test"))["tasks"] == 2 + + +def test_speech_replay_does_not_revalidate_an_unavailable_model(tmp_path): + native = FakeNative(tmp_path) + service = configured_service(native) + first = _run(service.submit(speech_command())) + native.preflight_error = RuntimeError("Model is now unavailable") + assert _run(service.submit(speech_command()))["receipt"] == first["receipt"] + changed = speech_command() + changed["input"]["params"]["prompt"] += " Changed" + with pytest.raises(HTTPException) as caught: + _run(service.submit(changed)) + assert caught.value.status_code == 409 + assert len(native.dispatch_calls) == 1 + + +def test_speech_restart_rebuilds_existing_recovery_without_dispatch(tmp_path): + first_native = FakeNative(tmp_path) + first = _run(configured_service(first_native).submit(speech_command())) + restarted = FakeNative(tmp_path, interrupt_stale=True) + service = configured_service(restarted) + service.restore_recovery(["speech-test"]) + assert len(restarted.persist_calls) == 1 + assert restarted.dispatch_calls == [] + record = restarted.persist_calls[0] + assert record["params"]["_tts_original_prompt"] == speech_command()["input"]["params"]["prompt"] + assert service.filter_recovery([record]) == [record] + replay = _run(service.submit(speech_command())) + assert replay["receipt"] == first["receipt"] + assert service.receipt("speech-test", "speech-test-intent")["task"]["status"] == "interrupted" + + +def test_runtime_without_speech_adapter_does_not_adopt_its_recovery(tmp_path): + _run(configured_service(FakeNative(tmp_path)).submit(speech_command())) + restarted = FakeNative(tmp_path, interrupt_stale=True) + restarted.service().restore_recovery(["speech-test"]) + assert restarted.persist_calls == [] + assert restarted.dispatch_calls == [] + + +def test_browser_catalog_matches_executable_speech_contract(): + path = Path(__file__).resolve().parents[1] / "ui/src/api/speechCommandCatalog.json" + projection = json.loads(path.read_text(encoding="utf-8")) + assert projection == {"version": 2, "operations": [speech_command_catalog()], "studio": studio_speech_schema()} diff --git a/tests/test_studio_speech_preparation.py b/tests/test_studio_speech_preparation.py new file mode 100644 index 000000000..19e8d63e2 --- /dev/null +++ b/tests/test_studio_speech_preparation.py @@ -0,0 +1,349 @@ +"""Provider-free tests for Studio speech model/resource preparation.""" + +from copy import deepcopy +import json +from pathlib import Path + +import pytest +from fastapi import HTTPException + +from services.studio_speech_preparation import prepare_studio_speech + + +FIXTURE = Path(__file__).parent / "fixtures" / "studio_speech_native_request.json" + +KUGEL_DEFINITION = { + "audio_only": True, + "image_outputs": False, + "guidance_max_phases": 1, + "no_negative_prompt": True, + "inference_steps": False, + "temperature": True, + "duration_slider": {"min": 1, "max": 600, "default": 20}, + "audio_prompt_type_sources": {"selection": ["", "A", "AB"], "default": ""}, + "audio_mode_from_voice_count": True, + "max_voice_count": 6, + "custom_settings": [{"id": "auto_split_every_s", "type": "float", "min": 5, "max": 90}], +} + + +def base_params(**overrides): + params = json.loads(FIXTURE.read_text()) + params.update(overrides) + return params + + +class FakeResources: + def __init__(self, *, media_result=None, lora_result=None, media_error=None): + self.media_result = media_result + self.lora_result = lora_result if lora_result is not None else [] + self.media_error = media_error + self.media_calls = [] + self.lora_calls = [] + + def prepare_media(self, params): + self.media_calls.append(deepcopy(params)) + if self.media_error is not None: + raise self.media_error + return deepcopy(self.media_result if self.media_result is not None else (params, [])) + + def prepare_loras(self, params, definition): + self.lora_calls.append((deepcopy(params), deepcopy(definition))) + return deepcopy(self.lora_result) + + +def invoke( + params, + *, + definition=None, + downloaded=True, + resources=None, + policy_error=None, +): + definition = deepcopy(definition or KUGEL_DEFINITION) + resources = resources or FakeResources() + policy_calls = [] + definition_calls = [] + download_calls = [] + + def model_definition(model_type): + definition_calls.append(model_type) + return deepcopy(definition) + + def model_downloaded(model_type): + download_calls.append(model_type) + return downloaded + + def policy(workspace): + policy_calls.append(workspace) + if policy_error is not None: + raise policy_error + + result = prepare_studio_speech( + params, + model_definition=model_definition, + model_downloaded=model_downloaded, + resources=resources, + execution_policy=policy, + ) + return result, { + "resources": resources, + "policy_calls": policy_calls, + "definition_calls": definition_calls, + "download_calls": download_calls, + } + + +def assert_http_error(error_info, *, status=422): + assert error_info.value.status_code == status + assert error_info.value.detail["code"] == ( + "model_unavailable" if status == 409 else "invalid_studio_speech_input" + ) + return error_info.value.detail["message"] + + +def test_installed_speech_model_returns_detached_native_snapshot_and_resources(): + params = base_params() + before = deepcopy(params) + resources = FakeResources( + media_result=( + {**params, "duration_seconds": 20.0, "nested": {"keep": [1]}}, + [{"role": "audio_guide", "sha256": "audio-hash"}], + ), + lora_result=[{"role": "lora", "name": "voice-style", "sha256": "lora-hash"}], + ) + + (native, identities), calls = invoke(params, resources=resources) + + assert params == before + assert native == resources.media_result[0] + assert native is not resources.media_result[0] + assert identities == [ + {"role": "audio_guide", "sha256": "audio-hash"}, + {"role": "lora", "name": "voice-style", "sha256": "lora-hash"}, + ] + assert calls["policy_calls"] == ["speech-test"] + assert calls["definition_calls"] == ["kugelaudio_0_open"] + assert calls["download_calls"] == ["kugelaudio_0_open"] + assert resources.media_calls[0] == before + assert resources.lora_calls[0][0] == before + native["nested"]["keep"].append(2) + assert resources.media_result[0]["nested"] == {"keep": [1]} + assert params == before + + +@pytest.mark.parametrize("model_type", ["minimax_music3", "ace_step_v1", "yue", "mmaudio_v2", "unknown_audio"]) +def test_music_sfx_and_unknown_audio_models_are_not_speech(model_type): + resources = FakeResources() + params = base_params(model_type=model_type) + with pytest.raises(HTTPException) as error: + invoke(params, resources=resources) + message = assert_http_error(error) + assert "speech" in message.lower() or "music" in message.lower() + assert resources.media_calls == resources.lora_calls == [] + + +@pytest.mark.parametrize( + ("definition", "downloaded", "status"), + [ + ({**KUGEL_DEFINITION, "audio_only": False}, True, 422), + ({**KUGEL_DEFINITION, "image_outputs": True}, True, 422), + (KUGEL_DEFINITION, False, 409), + ], +) +def test_model_must_be_installed_audio_only_and_downloaded(definition, downloaded, status): + resources = FakeResources() + with pytest.raises(HTTPException) as error: + invoke(base_params(), definition=definition, downloaded=downloaded, resources=resources) + assert_http_error(error, status=status) + assert resources.media_calls == resources.lora_calls == [] + + +def test_model_mode_default_is_declared_by_model_and_invalid_mode_fails_early(): + definition = { + **KUGEL_DEFINITION, + "model_modes": {"choices": [("English", "en"), ("Spanish", "es")], "default": "en"}, + "audio_mode_from_voice_count": False, + } + params = base_params() + (native, _), _ = invoke(params, definition=definition) + assert native["model_mode"] == "en" + + bad = base_params(model_mode="de") + with pytest.raises(HTTPException) as error: + invoke(bad, definition=definition) + assert "model_mode" in assert_http_error(error) + + +def test_duration_zero_is_preserved_when_the_selected_model_declares_auto_duration(): + definition = { + **KUGEL_DEFINITION, + "audio_mode_from_voice_count": False, + "inference_steps": True, + "guidance_max_phases": 0, + "temperature": False, + "duration_slider": {"min": 0, "max": 60, "default": 0}, + "audio_prompt_type_sources": {"selection": ["", "A", "AB"], "default": ""}, + } + params = base_params( + model_type="dramabox_audio", + duration_seconds=0, + guidance_phases=0, + ) + params.pop("temperature") + (native, _), _ = invoke(params, definition=definition) + assert native["duration_seconds"] == 0 + + +def test_locked_scenema_step_count_is_preserved_even_without_an_editable_step_control(): + definition = { + **KUGEL_DEFINITION, + "audio_mode_from_voice_count": False, + "inference_steps": False, + "lock_inference_steps": True, + "guidance_max_phases": 0, + "temperature": False, + "duration_slider": {"min": 1, "max": 1800, "default": 120}, + "audio_prompt_type_sources": {"selection": ["", "A2", "AB2"], "default": ""}, + } + params = base_params(guidance_phases=0, num_inference_steps=8, temperature=None) + (native, _), _ = invoke(params, definition=definition) + assert native["num_inference_steps"] == 8 + + +@pytest.mark.parametrize( + ("params", "definition", "needle"), + [ + (base_params(negative_prompt="forbidden"), KUGEL_DEFINITION, "negative_prompt"), + (base_params(num_inference_steps=2), KUGEL_DEFINITION, "num_inference_steps"), + (base_params(guidance_phases=2), KUGEL_DEFINITION, "guidance_phases"), + ( + base_params(temperature=1), + {**KUGEL_DEFINITION, "temperature": False}, + "temperature", + ), + ( + base_params(top_k=10), + KUGEL_DEFINITION, + "top_k", + ), + ], +) +def test_declared_native_capabilities_fail_before_resource_inspection(params, definition, needle): + resources = FakeResources() + with pytest.raises(HTTPException) as error: + invoke(params, definition=definition, resources=resources) + assert needle in assert_http_error(error) + assert resources.media_calls == resources.lora_calls == [] + + +def test_audio_prompt_selector_and_references_must_be_coherent(): + missing = base_params(audio_prompt_type="A", _tts_voice_count=1) + with pytest.raises(HTTPException) as error: + invoke(missing) + assert "audio_guide" in assert_http_error(error) + + orphan = base_params(audio_prompt_type="", _tts_voice_count=1, audio_guide="/api/v1/uploads/voice.wav") + with pytest.raises(HTTPException) as error: + invoke(orphan) + assert "audio_prompt_type" in assert_http_error(error) + + missing_second = base_params( + audio_prompt_type="AB", _tts_voice_count=2, + audio_guide="/api/v1/uploads/one.wav", + ) + with pytest.raises(HTTPException) as error: + invoke(missing_second) + assert "audio_guide2" in assert_http_error(error) + + valid = base_params( + audio_prompt_type="AB", _tts_voice_count=2, + audio_guide="/api/v1/uploads/one.wav", + audio_guide2="/api/v1/file/two.wav?workspace=speech-test", + ) + resources = FakeResources() + (native, _), _ = invoke(valid, resources=resources) + assert native["audio_guide"] == valid["audio_guide"] + assert native["audio_guide2"] == valid["audio_guide2"] + assert resources.media_calls[0]["audio_guide2"] == valid["audio_guide2"] + + +def test_voice_mode_modifiers_are_preserved_while_the_declared_base_mode_is_checked(): + definition = { + **KUGEL_DEFINITION, + "audio_prompt_type_sources": { + "selection": ["", "A2", "AB2"], + "default": "", + "custom_flags": {"2": "SeedVC"}, + }, + "audio_mode_from_voice_count": True, + "max_voice_count": 2, + } + params = base_params( + audio_prompt_type="AB2NV", + _tts_voice_count=2, + audio_guide="/api/v1/uploads/one.wav", + audio_guide2="/api/v1/uploads/two.wav", + ) + (native, _), _ = invoke(params, definition=definition) + assert native["audio_prompt_type"] == "AB2NV" + + +def test_voice_count_cannot_exceed_model_declared_limit(): + params = base_params(_tts_voice_count=3, audio_prompt_type="AB") + definition = {**KUGEL_DEFINITION, "max_voice_count": 2} + with pytest.raises(HTTPException) as error: + invoke(params, definition=definition) + assert "voice" in assert_http_error(error).lower() + + +def test_custom_settings_use_model_metadata_for_keys_types_and_ranges(): + bad_range = base_params(custom_settings={"auto_split_every_s": 4}) + with pytest.raises(HTTPException) as error: + invoke(bad_range) + assert "minimum" in assert_http_error(error) + + bad_unknown = base_params(custom_settings={"not_declared": 1}) + with pytest.raises(HTTPException) as error: + invoke(bad_unknown) + assert "unknown" in assert_http_error(error) + + valid = base_params(custom_settings={"auto_split_every_s": 5.0}) + (native, _), _ = invoke(valid) + assert native["custom_settings"] == valid["custom_settings"] + + +def test_loras_are_rejected_without_explicit_audio_lora_capability_and_resolved_when_enabled(): + params = base_params(activated_loras=["voice-style"]) + resources = FakeResources(lora_result=[{"role": "lora", "name": "voice-style"}]) + with pytest.raises(HTTPException) as error: + invoke(params, resources=resources) + assert "lora" in assert_http_error(error).lower() + assert resources.media_calls == resources.lora_calls == [] + + definition = {**KUGEL_DEFINITION, "enabled_audio_lora": True} + (native, identities), _ = invoke(params, definition=definition, resources=resources) + assert native["activated_loras"] == ["voice-style"] + assert identities == [{"role": "lora", "name": "voice-style"}] + assert len(resources.lora_calls) == 1 + + +def test_resource_failure_is_wrapped_and_loras_are_not_looked_up_after_media_failure(): + resources = FakeResources(media_error=ValueError("bad voice reference")) + with pytest.raises(HTTPException) as error: + invoke(base_params(), resources=resources) + assert "bad voice reference" in assert_http_error(error) + assert len(resources.media_calls) == 1 + assert resources.lora_calls == [] + + +def test_execution_policy_runs_before_model_lookup(): + resources = FakeResources() + with pytest.raises(HTTPException) as error: + invoke( + base_params(), + resources=resources, + policy_error=HTTPException(409, {"code": "execution_policy", "message": "busy"}), + ) + assert error.value.status_code == 409 + assert resources.media_calls == resources.lora_calls == [] diff --git a/tests/test_studio_speech_resources.py b/tests/test_studio_speech_resources.py new file mode 100644 index 000000000..3763125cc --- /dev/null +++ b/tests/test_studio_speech_resources.py @@ -0,0 +1,112 @@ +"""Canonical voice sources stay separate from the destination and native fallback.""" +from copy import deepcopy +import hashlib +import wave + +import pytest + +from services.studio_speech_resources import StudioSpeechResources +from services.asset_manifest import build_asset_manifest, write_asset_manifest +from services.wangp_submission import prepare_generation_inputs + + +def write_wave(path, frames=800): + with wave.open(str(path), "wb") as output: + output.setparams((1, 2, 8000, 0, "NONE", "not compressed")) + output.writeframes(b"\x01\x00" * frames) + + +@pytest.fixture +def sources(tmp_path): + folders = {name: tmp_path / name for name in ("uploads", "source", "destination", "default")} + for path in folders.values(): + path.mkdir() + resources = StudioSpeechResources( + workspace_dir=lambda name: folders[name], uploads_dir=lambda: folders["uploads"], + list_workspaces=lambda: [{"name": name} for name in ("source", "destination")], + lora_search_dirs=lambda _: [], lora_compatible=lambda *_: False, + ) + return folders, resources + + +@pytest.mark.parametrize("new_family", [False, True]) +def test_voice_references_use_declared_source_across_native_preparation(sources, new_family): + folders, resources = sources + write_wave(folders["source"] / "same.wav", 800) + write_wave(folders["destination"] / "same.wav", 1600) + write_wave(folders["uploads"] / "voice.wav", 2400) + params = {"generation_mode": "audio", "image_mode": 0, "workspace": "destination", + "audio_guide": "/api/v1/file/same.wav?workspace=source", + "audio_guide6": "/api/v1/uploads/voice.wav", "audio_guide2": ""} + original = deepcopy(params) + working, identities = resources.prepare_media(params) + prepare_generation_inputs(working, {"audio_only": True, "wangp_1272": new_family}, "destination", + workspace_dir=folders["destination"], uploads_dir=folders["uploads"], + prepared_speech=True) + assert params == original + assert working["audio_guide"] == str(folders["source"] / "same.wav") + assert working["audio_guide6"] == str(folders["uploads"] / "voice.wav") + assert working["audio_guide2"] == "" + assert [item["workspace"] for item in identities] == ["source", "__uploads__"] + assert identities[0]["duration_seconds"] == 0.1 + assert identities[0]["sha256"] == hashlib.sha256((folders["source"] / "same.wav").read_bytes()).hexdigest() + + +def test_missing_source_does_not_adopt_same_named_destination(sources): + folders, resources = sources + write_wave(folders["destination"] / "voice.wav") + with pytest.raises(ValueError): + resources.prepare_media({"audio_guide": "/api/v1/file/voice.wav?workspace=source"}) + + +def test_non_audio_source_is_rejected(sources): + folders, resources = sources + (folders["source"] / "bad.wav").write_text("not audio") + with pytest.raises(ValueError): + resources.prepare_media({"audio_guide": "/api/v1/file/bad.wav?workspace=source"}) + + +def test_symlink_escape_is_rejected_before_probe(sources, tmp_path): + folders, resources = sources + write_wave(tmp_path / "outside.wav") + (folders["source"] / "voice.wav").symlink_to(tmp_path / "outside.wav") + with pytest.raises(ValueError): + resources.prepare_media({"audio_guide": "/api/v1/file/voice.wav?workspace=source"}) + + +def test_json_flag_cannot_authorize_prepared_voice_paths(sources, tmp_path): + folders, _ = sources + outside = tmp_path / "outside.wav" + write_wave(outside) + with pytest.raises(ValueError): + prepare_generation_inputs({"generation_mode": "audio", "audio_guide": str(outside), + "prepared_studio_speech": True}, + {"returns_audio": True, "wangp_1272": True}, "destination", + workspace_dir=folders["destination"], uploads_dir=folders["uploads"]) + + +@pytest.mark.parametrize("field", ["audio_guide", "audio_guide2", "audio_guide3", "audio_guide4", "audio_guide5", "audio_guide6"]) +def test_every_native_voice_slot_is_resolved_without_the_trusted_adapter(sources, field): + folders, _ = sources + write_wave(folders["uploads"] / "voice.wav") + working = {"generation_mode": "audio", field: "/api/v1/uploads/voice.wav"} + prepare_generation_inputs(working, {"audio_only": True, "wangp_1272": True}, "destination", + workspace_dir=folders["destination"], uploads_dir=folders["uploads"]) + assert working[field] == str(folders["uploads"] / "voice.wav") + + +@pytest.mark.parametrize("reference", ["asset_voice", "/api/v1/assets/asset_voice"]) +def test_audio_asset_identity_preserves_source_and_rejects_ambiguous_locations(sources, reference): + folders, resources = sources + source = folders["source"] / "voice.wav" + write_wave(source) + write_asset_manifest(source, build_asset_manifest(source, asset_id="asset_voice", tool="studio")) + working, identities = resources.prepare_media({"audio_guide": reference}) + assert working["audio_guide"] == str(source) + assert identities[0]["workspace"] == "source" + assert identities[0]["url"] == reference + destination = folders["destination"] / "voice.wav" + write_wave(destination) + write_asset_manifest(destination, build_asset_manifest(destination, asset_id="asset_voice", tool="studio")) + with pytest.raises(ValueError, match="multiple locations"): + resources.prepare_media({"audio_guide": reference}) diff --git a/ui/src/api/generationCommandClient.ts b/ui/src/api/generationCommandClient.ts new file mode 100644 index 000000000..79a319a0f --- /dev/null +++ b/ui/src/api/generationCommandClient.ts @@ -0,0 +1,719 @@ +import { BASE } from './http' +import { stableSerialize } from '../lib/commandContract' +import type { GenerationSubmissionContext } from '../features/studio/generationProvenance' + +/** + * Durable client-side transport shared by the typed Studio generation + * operations. A command-specific module owns its closed schema and passes + * only a detached validator here; this module owns persistence, ACK ordering, + * retries and receipt validation. + */ +export interface GenerationCommandInput { + workspace: string + [key: string]: unknown +} + +export interface GenerationCommandLike { + version: number + operation: string + intent_id: string + input: { workspace: string } +} + +export interface GenerationTaskResult { + job_id: string + task_id: string + workspace: string + status: 'queued' + root_task_id?: string | null +} + +export interface GenerationReceiptLike { + version: 1 + commandId: string + operation: string + status: 'queued' + entities: unknown[] + artifacts: unknown[] + taskIds: string[] + pipelineIds: string[] + result: GenerationTaskResult + replayed?: boolean + commandVersion?: 2 + contentFingerprint?: string + fingerprintVersion?: 2 +} + +export interface GenerationCommandErrorOptions { + status?: number + uncertain?: boolean + code?: string +} + +export class GenerationCommandError extends Error { + readonly intentId: string + readonly workspace: string + readonly status?: number + readonly uncertain: boolean + readonly code: string + + constructor( + message: string, + intentId: string, + workspace: string, + options: GenerationCommandErrorOptions = {}, + ) { + super(message) + this.name = 'GenerationCommandError' + this.intentId = intentId + this.workspace = workspace + this.status = options.status + this.uncertain = options.uncertain ?? false + this.code = options.code ?? 'generation_command_failed' + } +} + +type CommandErrorConstructor = new ( + message: string, + intentId: string, + workspace: string, + options?: GenerationCommandErrorOptions, +) => GenerationCommandError + +export interface GenerationCommandClientConfig< + Command extends GenerationCommandLike, + Receipt extends GenerationReceiptLike, +> { + /** Stable storage namespace. It must include the versioned operation family. */ + storagePrefix: string + contextStoragePrefix: string + pendingChangedEvent: string + operation: string + label: string + /** A v1 fallback keeps legacy receipt reads compatible when no hint exists. */ + receiptFallbackVersion?: number + detach: (value: unknown) => Command + errorClass?: CommandErrorConstructor + /** Build a receipt type without exposing a second transport implementation. */ + castReceipt?: (value: GenerationReceiptLike) => Receipt +} + +export interface SubmitGenerationCommandOptions { + /** UI attribution is transport metadata; it never enters the command hash. */ + submissionContext?: GenerationSubmissionContext + /** Runs after the durable pending hint and before the network POST. */ + onSnapshotReady?: (snapshot: Command) => void | Promise +} + +type StoredSubmissionContext = Pick + +interface ReceiptEnvelope { + receipt: unknown + replayed?: boolean + malformed?: boolean +} + +interface ReceiptContext { + version: number + intent_id: string + operation: string + input: Pick +} + +const MAX_INTENT_LENGTH = 160 +const MAX_WORKSPACE_LENGTH = 240 +const MAX_SUBMISSION_CONTEXT_ID_LENGTH = 200 +const SUBMISSION_ACTORS = new Set(['user', 'wizard', 'system', 'unknown']) + +function isRecord(value: unknown): value is Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false + const prototype = Object.getPrototypeOf(value) + return prototype === Object.prototype || prototype === null +} + +function errorFor( + config: GenerationCommandClientConfig, + message: string, + command: Pick & { input: Pick }, + options: GenerationCommandErrorOptions = {}, +): GenerationCommandError { + const ErrorClass = config.errorClass || GenerationCommandError + return new ErrorClass(message, command.intent_id, command.input.workspace, options) +} + +function requiredText(value: unknown, field: string, maximum: number): string { + if (typeof value !== 'string' || !value.trim()) throw new Error(`${field} must be a non-blank string`) + if (value.length > maximum) throw new Error(`${field} is too long`) + return value +} + +function storage(): Storage { + if (typeof globalThis.localStorage === 'undefined') { + throw new Error('localStorage is unavailable; command admission cannot be made safely') + } + return globalThis.localStorage +} + +function notifyPendingChanged(eventName: string): void { + if (typeof window === 'undefined' || typeof Event === 'undefined') return + window.dispatchEvent(new Event(eventName)) +} + +function contextPart(value: unknown, field: string): string | undefined { + if (value === undefined || value === null || value === '') return undefined + if (typeof value !== 'string' || !value.trim() || value.trim() !== value + || value.length > MAX_SUBMISSION_CONTEXT_ID_LENGTH) { + throw new Error(field + ' must be an exact non-blank string of at most 200 characters') + } + return value +} + +function validateSubmissionContext(context: GenerationSubmissionContext | undefined): StoredSubmissionContext | undefined { + if (!context) return undefined + if (!SUBMISSION_ACTORS.has(context.actor)) throw new Error('submissionContext.actor must be a known actor') + const workflowId = contextPart(context.workflowId, 'submissionContext.workflowId') + const runId = contextPart(context.runId, 'submissionContext.runId') + return { + actor: context.actor, + ...(workflowId !== undefined ? { workflowId } : {}), + ...(runId !== undefined ? { runId } : {}), + } +} + +function receiptError( + config: GenerationCommandClientConfig, + command: ReceiptContext, + message = 'Receipt could not be verified', +): GenerationCommandError { + return errorFor(config, `${message} for ${command.intent_id}`, command, { + status: 200, + uncertain: true, + code: 'invalid_receipt', + }) +} + +function pendingKey( + config: GenerationCommandClientConfig, + intentId: string, +): string { + return config.storagePrefix + intentId +} + +function pendingContextKey( + config: GenerationCommandClientConfig, + intentId: string, +): string { + return config.contextStoragePrefix + intentId +} + +function invalidStoredCommand( + config: GenerationCommandClientConfig, + intentId: string, +): GenerationCommandError { + return errorFor(config, `Stored ${config.label} command ${intentId} is invalid`, { + intent_id: intentId, + input: { workspace: '' }, + }, { code: 'invalid_pending_command' }) +} + +function readPending( + config: GenerationCommandClientConfig, + intentId: string, +): Command | null { + const raw = storage().getItem(pendingKey(config, intentId)) + if (raw == null) return null + try { + const command = config.detach(JSON.parse(raw)) + if (command.intent_id !== intentId) throw new Error('intent_id does not match storage key') + return command + } catch { + throw invalidStoredCommand(config, intentId) + } +} + +const STORED_CONTEXT_ENVELOPE_FIELDS = new Set(['version', 'intent_id', 'workspace', 'context']) +const STORED_CONTEXT_FIELDS = new Set(['actor', 'workflowId', 'runId']) + +function invalidStoredContext( + config: GenerationCommandClientConfig, + intentId: string, +): GenerationCommandError { + return errorFor(config, `Stored ${config.label} context ${intentId} is invalid`, { + intent_id: intentId, + input: { workspace: '' }, + }, { code: 'invalid_pending_context' }) +} + +function normalizeStoredContext(value: unknown): StoredSubmissionContext { + if (!isRecord(value)) throw new Error('context must be an object') + for (const key of Object.keys(value)) { + if (!STORED_CONTEXT_FIELDS.has(key)) throw new Error('context contains an unsupported field') + } + if (typeof value.actor !== 'string' || !SUBMISSION_ACTORS.has(value.actor)) throw new Error('context.actor is invalid') + const workflowId = contextPart(value.workflowId, 'context.workflowId') + const runId = contextPart(value.runId, 'context.runId') + return { + actor: value.actor as StoredSubmissionContext['actor'], + ...(workflowId !== undefined ? { workflowId } : {}), + ...(runId !== undefined ? { runId } : {}), + } +} + +function readPendingContext( + config: GenerationCommandClientConfig, + command: Command, +): StoredSubmissionContext | null { + const raw = storage().getItem(pendingContextKey(config, command.intent_id)) + if (raw == null) return null + try { + const value: unknown = JSON.parse(raw) + if (!isRecord(value) + || Object.keys(value).some(key => !STORED_CONTEXT_ENVELOPE_FIELDS.has(key)) + || value.version !== 1 + || value.intent_id !== command.intent_id + || value.workspace !== command.input.workspace) throw new Error('context envelope does not match command') + return normalizeStoredContext(value.context) + } catch { + throw invalidStoredContext(config, command.intent_id) + } +} + +function persistPendingContext( + config: GenerationCommandClientConfig, + command: Command, + context: StoredSubmissionContext | undefined, +): void { + const key = pendingContextKey(config, command.intent_id) + if (!context) { + if (storage().getItem(key) != null) storage().removeItem(key) + return + } + storage().setItem(key, stableSerialize({ + version: 1, + intent_id: command.intent_id, + workspace: command.input.workspace, + context, + })) +} + +function sameSubmissionContext(left: StoredSubmissionContext, right: StoredSubmissionContext): boolean { + return stableSerialize(left) === stableSerialize(right) +} + +function sameCommand(left: Command, right: Command): boolean { + return stableSerialize(left) === stableSerialize(right) +} + +function retainPending( + config: GenerationCommandClientConfig, + command: Command, + done = false, +): boolean { + const existing = readPending(config, command.intent_id) + if (existing && !sameCommand(existing, command)) { + throw errorFor(config, `intent_id ${command.intent_id} is already pending with a different command`, command, { + code: 'intent_conflict', + }) + } + const key = pendingKey(config, command.intent_id) + if (done) { + const current = readPending(config, command.intent_id) + if (current && sameCommand(current, command)) { + persistPendingContext(config, command, undefined) + storage().removeItem(key) + } else if (!current) { + persistPendingContext(config, command, undefined) + } + } else { + storage().setItem(key, stableSerialize(command)) + } + notifyPendingChanged(config.pendingChangedEvent) + return existing != null +} + +function forgetPending( + config: GenerationCommandClientConfig, + command: Command, +): void { + try { retainPending(config, command, true) } catch { /* cleanup is best effort after a confirmed receipt */ } +} + +function errorDetail(value: unknown): string | undefined { + if (typeof value === 'string' && value.trim()) return value + if (!isRecord(value)) return undefined + const detail = value.detail + if (typeof detail === 'string' && detail.trim()) return detail + if (isRecord(detail) && typeof detail.message === 'string' && detail.message.trim()) return detail.message + if (typeof value.message === 'string' && value.message.trim()) return value.message + return undefined +} + +async function responseError( + config: GenerationCommandClientConfig, + response: Response, + command: Pick & { input: Pick }, +): Promise { + const payload = await response.json().catch(() => undefined) + return errorFor(config, errorDetail(payload) || `${config.label} command failed (${response.status})`, command, { + status: response.status, + uncertain: response.status >= 500, + code: 'http_error', + }) +} + +function unwrapReceipt(value: unknown): ReceiptEnvelope { + if (isRecord(value) && 'receipt' in value) { + if ('replayed' in value && typeof value.replayed !== 'boolean') return { receipt: value.receipt, malformed: true } + return { + receipt: value.receipt, + replayed: typeof value.replayed === 'boolean' ? value.replayed : undefined, + } + } + return { receipt: value } +} + +function receiptReplay( + config: GenerationCommandClientConfig, + value: Record, + command: ReceiptContext, + fallback?: boolean, +): boolean | undefined { + if ('replayed' in value && typeof value.replayed !== 'boolean') throw receiptError(config, command) + return typeof value.replayed === 'boolean' ? value.replayed : fallback +} + +function receiptV2Metadata( + config: GenerationCommandClientConfig, + value: Record, + command: ReceiptContext, +): Pick { + const fields = ['commandVersion', 'contentFingerprint', 'fingerprintVersion'] as const + const present = fields.filter(field => field in value) + if (present.length !== 0 && present.length !== fields.length) { + throw receiptError(config, command, 'Receipt fingerprint metadata is incomplete') + } + if (present.length === 0) { + if (command.version === 2) throw receiptError(config, command, 'The v2 receipt is missing its content fingerprint') + return {} + } + if (value.commandVersion !== 2 || value.fingerprintVersion !== 2 + || typeof value.contentFingerprint !== 'string' + || !/^[a-f0-9]{64}$/.test(value.contentFingerprint)) { + throw receiptError(config, command, 'Receipt fingerprint metadata is invalid') + } + return { commandVersion: 2, contentFingerprint: value.contentFingerprint, fingerprintVersion: 2 } +} + +function validTaskResult(value: unknown, workspace: string, taskIds: unknown[]): value is GenerationTaskResult { + return isRecord(value) + && typeof value.job_id === 'string' && value.job_id.length > 0 + && typeof value.task_id === 'string' && value.task_id.length > 0 + && taskIds.length === 1 && taskIds[0] === value.task_id + && value.workspace === workspace && value.status === 'queued' +} + +function validateReceipt( + config: GenerationCommandClientConfig, + value: unknown, + command: ReceiptContext, + replayed?: boolean, +): Receipt { + if (!isRecord(value)) throw receiptError(config, command) + const outerReplayed = receiptReplay(config, value, command, replayed) + const metadata = receiptV2Metadata(config, value, command) + const taskIds = value.taskIds + if (value.version !== 1 + || value.commandId !== command.intent_id + || value.operation !== command.operation + || value.status !== 'queued' + || !Array.isArray(value.entities) + || !Array.isArray(value.artifacts) + || !Array.isArray(taskIds) + || !Array.isArray(value.pipelineIds) + || !validTaskResult(value.result, command.input.workspace, taskIds)) { + throw receiptError(config, command) + } + const receipt: GenerationReceiptLike = { + version: 1, + commandId: command.intent_id, + operation: command.operation, + status: 'queued', + entities: JSON.parse(stableSerialize(value.entities)) as unknown[], + artifacts: JSON.parse(stableSerialize(value.artifacts)) as unknown[], + taskIds: [...taskIds] as string[], + pipelineIds: [...value.pipelineIds] as string[], + result: JSON.parse(stableSerialize(value.result)) as GenerationTaskResult, + ...metadata, + } + if (outerReplayed !== undefined) receipt.replayed = outerReplayed + return config.castReceipt ? config.castReceipt(receipt) : receipt as Receipt +} + +function requestedContext( + config: GenerationCommandClientConfig, + snapshot: Command, + context: GenerationSubmissionContext | undefined, +): StoredSubmissionContext | undefined { + try { + return validateSubmissionContext(context) + } catch (error) { + throw errorFor(config, error instanceof Error ? error.message : 'The submission context is invalid', snapshot, { + code: 'invalid_submission_context', + }) + } +} + +function assertContextMatches( + config: GenerationCommandClientConfig, + snapshot: Command, + stored: StoredSubmissionContext | null, + requested: StoredSubmissionContext | undefined, +): void { + if (stored && requested && !sameSubmissionContext(stored, requested)) { + throw errorFor(config, `intent_id ${snapshot.intent_id} is already attributed to a different UI context`, snapshot, { + code: 'submission_context_conflict', uncertain: true, + }) + } +} + +function preparePendingSubmission( + config: GenerationCommandClientConfig, + snapshot: Command, + requested: StoredSubmissionContext | undefined, +): { recovering: boolean; submissionContext?: StoredSubmissionContext } { + let recovering = false + try { + const existing = readPending(config, snapshot.intent_id) + if (existing && !sameCommand(existing, snapshot)) { + throw errorFor(config, `intent_id ${snapshot.intent_id} is already pending with a different command`, snapshot, { + code: 'intent_conflict', + }) + } + recovering = existing !== null + if (!recovering) persistPendingContext(config, snapshot, requested) + recovering = retainPending(config, snapshot) + const stored = recovering ? readPendingContext(config, snapshot) : null + assertContextMatches(config, snapshot, stored, requested) + if (recovering && !stored && requested) persistPendingContext(config, snapshot, requested) + return { recovering, submissionContext: stored || requested } + } catch (error) { + if (error instanceof GenerationCommandError) throw error + throw errorFor(config, error instanceof Error ? error.message : `Could not persist ${config.label} command`, snapshot, { + code: 'pending_storage_failed', uncertain: recovering, + }) + } +} + +async function presentSnapshot( + config: GenerationCommandClientConfig, + snapshot: Command, + recovering: boolean, + hook: SubmitGenerationCommandOptions['onSnapshotReady'], +): Promise { + if (!hook) return + try { + await hook(config.detach(snapshot)) + } catch (error) { + if (!recovering) forgetPending(config, snapshot) + throw errorFor(config, error instanceof Error ? error.message : `${config.label} command could not be presented`, snapshot, { + code: 'snapshot_hook_failed', + }) + } +} + +async function postCommand( + config: GenerationCommandClientConfig, + command: Command, + submissionContext?: StoredSubmissionContext, +): Promise { + let response: Response + try { + const headers: Record = { 'Content-Type': 'application/json' } + if (submissionContext) { + headers['X-Hocus-UI-Surface'] = submissionContext.actor === 'wizard' ? 'wizard' : 'studio' + const context: Record = {} + const workflowId = contextPart(submissionContext.workflowId, 'submissionContext.workflowId') + const runId = contextPart(submissionContext.runId, 'submissionContext.runId') + if (workflowId !== undefined) context.workflowId = workflowId + if (runId !== undefined) context.runId = runId + if (Object.keys(context).length > 0) headers['X-Hocus-UI-Context'] = JSON.stringify(context) + } + response = await fetch(`${BASE}/api/v1/generation/commands`, { + method: 'POST', + headers, + body: JSON.stringify(command), + }) + } catch (error) { + throw errorFor(config, error instanceof Error ? error.message : `${config.label} request failed`, command, { + uncertain: true, code: 'transport_uncertain', + }) + } + if (!response.ok) throw await responseError(config, response, command) + try { + return await response.json() + } catch { + throw errorFor(config, `${config.label} response could not be decoded for ${command.intent_id}`, command, { + status: response.status, uncertain: true, code: 'invalid_response', + }) + } +} + +function receiptCommandContext( + config: GenerationCommandClientConfig, + intentId: string, + workspace: string, +): ReceiptContext { + const fallback: ReceiptContext = { + version: config.receiptFallbackVersion ?? 1, + intent_id: intentId, + operation: config.operation, + input: { workspace }, + } + try { + const pending = readPending(config, intentId) + return pending && pending.input.workspace === workspace ? pending : fallback + } catch { + return fallback + } +} + +async function requestReceipt( + config: GenerationCommandClientConfig, + workspace: string, + intentId: string, +): Promise { + const command = { intent_id: intentId, input: { workspace } } + try { + const response = await fetch(`${BASE}/api/v1/generation/commands/receipt?workspace=${encodeURIComponent(workspace)}&intent_id=${encodeURIComponent(intentId)}`) + if (!response.ok) throw await responseError(config, response, command) + return response + } catch (error) { + if (error instanceof GenerationCommandError) throw error + throw errorFor(config, error instanceof Error ? error.message : `${config.label} receipt request failed`, command, { + uncertain: true, code: 'transport_uncertain', + }) + } +} + +async function decodeReceiptResponse( + config: GenerationCommandClientConfig, + response: Response, + command: ReceiptContext, +): Promise { + try { + const payload = unwrapReceipt(await response.json()) + if (payload.malformed) throw receiptError(config, command) + return validateReceipt(config, payload.receipt, command, payload.replayed) + } catch (error) { + if (error instanceof GenerationCommandError) throw error + throw errorFor(config, error instanceof Error ? error.message : `${config.label} receipt could not be verified`, command, { + status: 200, uncertain: true, code: 'invalid_receipt', + }) + } +} + +function clearPendingReceipt( + config: GenerationCommandClientConfig, + intentId: string, + workspace: string, +): void { + try { + const pending = readPending(config, intentId) + if (pending && pending.input.workspace === workspace) forgetPending(config, pending) + } catch { /* A valid server receipt remains authoritative. */ } +} + +function isDefinitiveClientError(error: GenerationCommandError): boolean { + return error.status !== undefined && error.status >= 400 && error.status < 500 +} + +function normalizeFailure( + config: GenerationCommandClientConfig, + error: unknown, + snapshot: Command, + recovering: boolean, +): GenerationCommandError { + const commandError = error instanceof GenerationCommandError + ? error + : errorFor(config, error instanceof Error ? error.message : `${config.label} command failed`, snapshot, { + uncertain: true, code: 'unknown_failure', + }) + if (!recovering && isDefinitiveClientError(commandError)) forgetPending(config, snapshot) + return errorFor(config, commandError.message, snapshot, { + status: commandError.status, + uncertain: commandError.uncertain || recovering, + code: commandError.code, + }) +} + +export interface GenerationCommandClient { + newIntentId: () => string + pendingCommands: (workspace?: string) => Command[] + pendingCommand: (intentId: string, workspace?: string) => Command | null + submit: (command: Command, options?: SubmitGenerationCommandOptions) => Promise + fetchReceipt: (workspace: string, intentId: string) => Promise + subscribe: (callback: () => void) => () => void +} + +export function createGenerationCommandClient( + config: GenerationCommandClientConfig, +): GenerationCommandClient { + const newIntentId = () => globalThis.crypto?.randomUUID?.() + || `${config.operation.replace(/[^a-z0-9]+/gi, '-')}-${Date.now()}-${Math.random().toString(36).slice(2)}` + + const pendingCommands = (workspace?: string): Command[] => { + const pending: Command[] = [] + const area = storage() + for (let index = 0; index < area.length; index += 1) { + const key = area.key(index) + if (!key?.startsWith(config.storagePrefix)) continue + const intentId = key.slice(config.storagePrefix.length) + const command = readPending(config, intentId) + if (!command || (workspace !== undefined && command.input.workspace !== workspace)) continue + pending.push(command) + } + return pending.sort((left, right) => left.intent_id.localeCompare(right.intent_id)) + } + + const pendingCommand = (intentId: string, workspace?: string): Command | null => { + const command = readPending(config, intentId) + if (!command || (workspace !== undefined && command.input.workspace !== workspace)) return null + return command + } + + const submit = async (command: Command, options: SubmitGenerationCommandOptions = {}): Promise => { + const snapshot = config.detach(command) + const context = requestedContext(config, snapshot, options.submissionContext) + const prepared = preparePendingSubmission(config, snapshot, context) + await presentSnapshot(config, snapshot, prepared.recovering, options.onSnapshotReady) + try { + const envelope = unwrapReceipt(await postCommand(config, snapshot, prepared.submissionContext)) + if (envelope.malformed) throw receiptError(config, snapshot) + const receipt = validateReceipt(config, envelope.receipt, snapshot, envelope.replayed) + forgetPending(config, snapshot) + return receipt + } catch (error) { + throw normalizeFailure(config, error, snapshot, prepared.recovering) + } + } + + const fetchReceipt = async (workspace: string, intentId: string): Promise => { + requiredText(workspace, 'workspace', MAX_WORKSPACE_LENGTH) + requiredText(intentId, 'intent_id', MAX_INTENT_LENGTH) + const context = receiptCommandContext(config, intentId, workspace) + const receipt = await decodeReceiptResponse(config, await requestReceipt(config, workspace, intentId), context) + clearPendingReceipt(config, intentId, workspace) + return receipt + } + + const subscribe = (callback: () => void): (() => void) => { + window.addEventListener(config.pendingChangedEvent, callback) + window.addEventListener('storage', callback) + return () => { + window.removeEventListener(config.pendingChangedEvent, callback) + window.removeEventListener('storage', callback) + } + } + + return { newIntentId, pendingCommands, pendingCommand, submit, fetchReceipt, subscribe } +} diff --git a/ui/src/api/imageCommandCatalog.json b/ui/src/api/imageCommandCatalog.json index 4c763ce0f..bdf6e2495 100644 --- a/ui/src/api/imageCommandCatalog.json +++ b/ui/src/api/imageCommandCatalog.json @@ -1646,7 +1646,7 @@ "version": 1, "domain": "studio", "mutation": false, - "description": "Read an immutable image admission and its current canonical task in the exact original output workspace.", + "description": "Read an immutable generation admission and its current canonical task in the exact original output workspace.", "inputSchema": { "type": "object", "additionalProperties": false, diff --git a/ui/src/api/imageGenerationCommands.ts b/ui/src/api/imageGenerationCommands.ts index 03f112a80..9babfb38b 100644 --- a/ui/src/api/imageGenerationCommands.ts +++ b/ui/src/api/imageGenerationCommands.ts @@ -1,6 +1,12 @@ -import { BASE } from './http' import { stableSerialize } from '../lib/commandContract' -import type { GenerationSubmissionContext } from '../features/studio/generationProvenance' +import { + createGenerationCommandClient, + GenerationCommandError, + type GenerationReceiptLike, + type GenerationTaskResult, + type GenerationCommandErrorOptions, + type SubmitGenerationCommandOptions, +} from './generationCommandClient' import { assertStudioImageGenerationCommand, detachedStudioImageGenerationCommand, @@ -18,7 +24,7 @@ export type { StudioImageParams, } from '../features/studio/generationSpec' -/** The first shared generation vertical deliberately exposes image only. */ +/** The legacy image command remains supported beside the typed Studio command. */ export const IMAGE_GENERATION_OPERATION = 'generation.image' as const export const IMAGE_GENERATION_SCHEMA_VERSION = 1 as const @@ -45,69 +51,33 @@ export interface ImageGenerationCommandV1 { export type ImageGenerationCommandV2 = StudioImageGenerationCommand export type ImageGenerationCommand = ImageGenerationCommandV1 | ImageGenerationCommandV2 -export interface ImageGenerationTaskResult { - job_id: string - task_id: string - workspace: string - status: 'queued' - root_task_id?: string | null -} +export type ImageGenerationTaskResult = GenerationTaskResult -export interface ImageGenerationReceipt { +export interface ImageGenerationReceipt extends GenerationReceiptLike { version: typeof IMAGE_GENERATION_SCHEMA_VERSION - commandId: string operation: typeof IMAGE_GENERATION_OPERATION - status: 'queued' - entities: unknown[] - artifacts: unknown[] - taskIds: string[] - pipelineIds: string[] result: ImageGenerationTaskResult - replayed?: boolean - commandVersion?: 2 - contentFingerprint?: string - fingerprintVersion?: 2 } -/** - * Recovery metadata is deliberately smaller than the caller context. It is - * stored beside the immutable command hint and only drives declared UI - * attribution headers; it is never part of the command or its fingerprint. - */ -type StoredSubmissionContext = Pick - -export class ImageGenerationCommandError extends Error { - readonly intentId: string - readonly workspace: string - readonly status?: number - readonly uncertain: boolean - readonly code: string - +export class ImageGenerationCommandError extends GenerationCommandError { constructor( message: string, intentId: string, workspace: string, - options: { status?: number; uncertain?: boolean; code?: string } = {}, + options: GenerationCommandErrorOptions = {}, ) { - super(message) + super(message, intentId, workspace, { + ...options, + code: options.code ?? 'image_generation_command_failed', + }) this.name = 'ImageGenerationCommandError' - this.intentId = intentId - this.workspace = workspace - this.status = options.status - this.uncertain = options.uncertain ?? false - this.code = options.code ?? 'image_generation_command_failed' } } -const PENDING_KEY_PREFIX = 'hocuspocus.generation.image-commands.v1:' -const PENDING_CONTEXT_KEY_PREFIX = 'hocuspocus.generation.image-command-context.v1:' -const PENDING_CHANGED_EVENT = 'hocuspocus:generation-image-commands-changed' const MAX_INTENT_LENGTH = 160 const MAX_ID_LENGTH = 240 const MAX_PROMPT_LENGTH = 200_000 const MAX_RESOLUTION_LENGTH = 128 -const MAX_SUBMISSION_CONTEXT_ID_LENGTH = 200 -const SUBMISSION_ACTORS = new Set(['user', 'wizard', 'system', 'unknown']) const INPUT_FIELDS = new Set([ 'workspace', @@ -195,186 +165,32 @@ export function assertImageGenerationCommandV1(value: unknown): asserts value is assertImageGenerationInput(value.input) } -function detachedCommand(value: unknown): ImageGenerationCommand { +function detachedImageCommand(value: unknown): ImageGenerationCommand { if (isRecord(value) && value.version === 2) { assertStudioImageGenerationCommand(value) return detachedStudioImageGenerationCommand(value) } assertImageGenerationCommandV1(value) - // stableSerialize validates the JSON boundary and gives the retry an - // immutable value-level snapshot. It never adds native defaults to input. - return JSON.parse(stableSerialize(value)) as ImageGenerationCommand -} - -function storage(): Storage { - if (typeof globalThis.localStorage === 'undefined') { - throw new Error('localStorage is unavailable; command admission cannot be made safely') - } - return globalThis.localStorage -} - -function notifyPendingChanged(): void { - if (typeof window === 'undefined' || typeof Event === 'undefined') return - window.dispatchEvent(new Event(PENDING_CHANGED_EVENT)) -} - -function pendingKey(intentId: string): string { - return PENDING_KEY_PREFIX + intentId -} - -function invalidStoredCommand(intentId: string): ImageGenerationCommandError { - return new ImageGenerationCommandError( - `Stored image generation command ${intentId} is invalid`, - intentId, - '', - { code: 'invalid_pending_command' }, - ) -} - -function readPending(intentId: string): ImageGenerationCommand | null { - const raw = storage().getItem(pendingKey(intentId)) - if (raw == null) return null - try { - const value: unknown = JSON.parse(raw) - const command = detachedCommand(value) - if (command.intent_id !== intentId) throw new Error('intent_id does not match storage key') - return command - } catch { - throw invalidStoredCommand(intentId) - } -} - -const STORED_CONTEXT_ENVELOPE_FIELDS = new Set(['version', 'intent_id', 'workspace', 'context']) -const STORED_CONTEXT_FIELDS = new Set(['actor', 'workflowId', 'runId']) - -function pendingContextKey(intentId: string): string { - return PENDING_CONTEXT_KEY_PREFIX + intentId -} - -function invalidStoredContext(intentId: string): ImageGenerationCommandError { - return new ImageGenerationCommandError( - `Stored image generation context ${intentId} is invalid`, - intentId, - '', - { code: 'invalid_pending_context' }, - ) -} - -function normalizeStoredSubmissionContext(value: unknown): StoredSubmissionContext { - if (!isRecord(value)) throw new Error('context must be an object') - for (const key of Object.keys(value)) { - if (!STORED_CONTEXT_FIELDS.has(key)) throw new Error('context contains an unsupported field') - } - if (typeof value.actor !== 'string' || !SUBMISSION_ACTORS.has(value.actor)) { - throw new Error('context.actor is invalid') - } - const workflowId = submissionContextPart(value.workflowId, 'context.workflowId') - const runId = submissionContextPart(value.runId, 'context.runId') - return { - actor: value.actor as StoredSubmissionContext['actor'], - ...(workflowId !== undefined ? { workflowId } : {}), - ...(runId !== undefined ? { runId } : {}), - } -} - -function readPendingContext(command: ImageGenerationCommand): StoredSubmissionContext | null { - const raw = storage().getItem(pendingContextKey(command.intent_id)) - if (raw == null) return null - try { - const value: unknown = JSON.parse(raw) - if (!isRecord(value) - || Object.keys(value).some(key => !STORED_CONTEXT_ENVELOPE_FIELDS.has(key)) - || value.version !== 1 - || value.intent_id !== command.intent_id - || value.workspace !== command.input.workspace) { - throw new Error('context envelope does not match its command') - } - return normalizeStoredSubmissionContext(value.context) - } catch { - throw invalidStoredContext(command.intent_id) - } -} - -function persistPendingContext( - command: ImageGenerationCommand, - context: StoredSubmissionContext | undefined, -): void { - const key = pendingContextKey(command.intent_id) - if (!context) { - // Avoid turning a no-op cleanup into a storage failure. This also keeps - // the legacy command-only path compatible with callers whose storage - // implementation rejects removeItem even when the key is absent. - if (storage().getItem(key) != null) storage().removeItem(key) - return - } - storage().setItem(key, stableSerialize({ - version: 1, - intent_id: command.intent_id, - workspace: command.input.workspace, - context, - })) -} - -function sameSubmissionContext( - left: StoredSubmissionContext, - right: StoredSubmissionContext, -): boolean { - return stableSerialize(left) === stableSerialize(right) -} - -function sameCommand(left: ImageGenerationCommand, right: ImageGenerationCommand): boolean { - return stableSerialize(left) === stableSerialize(right) -} - -function retainPending(command: ImageGenerationCommand, done = false): boolean { - const existing = readPending(command.intent_id) - if (existing && !sameCommand(existing, command)) { - throw new ImageGenerationCommandError( - `intent_id ${command.intent_id} is already pending with a different command`, - command.intent_id, - command.input.workspace, - { code: 'intent_conflict' }, - ) - } - const key = pendingKey(command.intent_id) - if (done) { - // A different tab may have replaced the value. Never erase that command - // while cleaning up a receipt for this one. - const current = readPending(command.intent_id) - if (current && sameCommand(current, command)) { - // Clear the sidecar first. If storage cleanup fails, retain the command - // hint so a confirmed result remains recoverable as before this sidecar - // existed. - persistPendingContext(command, undefined) - storage().removeItem(key) - } else if (!current) { - // The command may already have been removed by another tab after its - // receipt was confirmed. Its context key is still scoped by intent and - // can be cleaned without touching a replacement command. - persistPendingContext(command, undefined) - } - } else { - storage().setItem(key, stableSerialize(command)) - } - notifyPendingChanged() - return existing != null + return JSON.parse(stableSerialize(value)) as ImageGenerationCommandV1 } -function forgetPending(command: ImageGenerationCommand): void { - try { retainPending(command, true) } catch { /* A cleanup failure must not hide a confirmed server result. */ } -} +const imageCommandClient = createGenerationCommandClient({ + storagePrefix: 'hocuspocus.generation.image-commands.v1:', + contextStoragePrefix: 'hocuspocus.generation.image-command-context.v1:', + pendingChangedEvent: 'hocuspocus:generation-image-commands-changed', + operation: IMAGE_GENERATION_OPERATION, + label: 'Image generation', + receiptFallbackVersion: IMAGE_GENERATION_SCHEMA_VERSION, + detach: detachedImageCommand, + errorClass: ImageGenerationCommandError, + castReceipt: value => value as ImageGenerationReceipt, +}) -export function newImageGenerationIntentId(): string { - return globalThis.crypto?.randomUUID?.() - || `image-${Date.now()}-${Math.random().toString(36).slice(2)}` -} - -/** Require the caller to choose the intention; this function never invents one. */ export function createImageGenerationCommand( intentId: string, input: ImageGenerationInput, ): ImageGenerationCommandV1 { - return detachedCommand({ + return detachedImageCommand({ version: IMAGE_GENERATION_SCHEMA_VERSION, operation: IMAGE_GENERATION_OPERATION, intent_id: intentId, @@ -382,525 +198,16 @@ export function createImageGenerationCommand( }) as ImageGenerationCommandV1 } -export function pendingImageGenerationCommands(workspace?: string): ImageGenerationCommand[] { - const pending: ImageGenerationCommand[] = [] - const area = storage() - for (let index = 0; index < area.length; index += 1) { - const key = area.key(index) - if (!key?.startsWith(PENDING_KEY_PREFIX)) continue - const intentId = key.slice(PENDING_KEY_PREFIX.length) - const command = readPending(intentId) - if (!command || (workspace !== undefined && command.input.workspace !== workspace)) continue - pending.push(command) - } - return pending.sort((left, right) => left.intent_id.localeCompare(right.intent_id)) -} - -export function pendingImageGenerationCommand( - intentId: string, - workspace?: string, -): ImageGenerationCommand | null { - const command = readPending(intentId) - if (!command || (workspace !== undefined && command.input.workspace !== workspace)) return null - return command -} - -function errorDetail(value: unknown): string | undefined { - if (typeof value === 'string' && value.trim()) return value - if (!isRecord(value)) return undefined - const detail = value.detail - if (typeof detail === 'string' && detail.trim()) return detail - if (isRecord(detail) && typeof detail.message === 'string' && detail.message.trim()) return detail.message - if (typeof value.message === 'string' && value.message.trim()) return value.message - return undefined -} +export const newImageGenerationIntentId = imageCommandClient.newIntentId +export const pendingImageGenerationCommands = imageCommandClient.pendingCommands +export const pendingImageGenerationCommand = imageCommandClient.pendingCommand -async function responseError( - response: Response, - intentId: string, - workspace: string, -): Promise { - const payload = await response.json().catch(() => undefined) - return new ImageGenerationCommandError( - errorDetail(payload) || `Image generation command failed (${response.status})`, - intentId, - workspace, - { status: response.status, uncertain: response.status >= 500, code: 'http_error' }, - ) -} - -interface ReceiptEnvelope { - receipt: unknown - replayed?: boolean - malformed?: boolean -} - -function unwrapReceipt(value: unknown): ReceiptEnvelope { - if (isRecord(value) && 'receipt' in value) { - if ('replayed' in value && typeof value.replayed !== 'boolean') { - return { receipt: value.receipt, malformed: true } - } - return { - receipt: value.receipt, - replayed: typeof value.replayed === 'boolean' ? value.replayed : undefined, - } - } - return { receipt: value } -} - -type ReceiptContext = Pick & { - input: Pick -} - -function invalidReceipt(command: ReceiptContext, message = 'Receipt could not be verified'): ImageGenerationCommandError { - return new ImageGenerationCommandError( - `${message} for ${command.intent_id}`, - command.intent_id, - command.input.workspace, - { status: 200, uncertain: true, code: 'invalid_receipt' }, - ) -} - -function validTaskResult(value: unknown, workspace: string, taskIds: unknown[]): value is ImageGenerationTaskResult { - return isRecord(value) - && typeof value.job_id === 'string' && value.job_id.length > 0 - && typeof value.task_id === 'string' && value.task_id.length > 0 - && taskIds.length === 1 && taskIds[0] === value.task_id - && value.workspace === workspace && value.status === 'queued' -} - -function receiptReplay(value: Record, command: ReceiptContext, fallback?: boolean): boolean | undefined { - if ('replayed' in value && typeof value.replayed !== 'boolean') throw invalidReceipt(command) - return typeof value.replayed === 'boolean' ? value.replayed : fallback -} - -function receiptV2Metadata(value: Record, command: ReceiptContext): { - commandVersion?: 2 - contentFingerprint?: string - fingerprintVersion?: 2 -} { - const metadataFields = ['commandVersion', 'contentFingerprint', 'fingerprintVersion'] as const - const present = metadataFields.filter(field => field in value) - // A receipt is either the complete v1 shape or the complete v2 shape. A - // half-present fingerprint must never be treated as a legacy receipt. - if (present.length !== 0 && present.length !== metadataFields.length) { - throw invalidReceipt(command, 'Receipt fingerprint metadata is incomplete') - } - if (present.length === 0) { - if (command.version === 2) { - throw invalidReceipt(command, 'The v2 receipt is missing its content fingerprint') - } - return {} - } - if (value.commandVersion !== 2 || value.fingerprintVersion !== 2 - || typeof value.contentFingerprint !== 'string' - || !/^[a-f0-9]{64}$/.test(value.contentFingerprint)) { - throw invalidReceipt(command, 'Receipt fingerprint metadata is invalid') - } - return { - commandVersion: 2, - contentFingerprint: value.contentFingerprint, - fingerprintVersion: 2, - } -} - -function validateReceipt( - value: unknown, - command: ReceiptContext, - replayed?: boolean, -): ImageGenerationReceipt { - if (!isRecord(value)) throw invalidReceipt(command) - const outerReplayed = receiptReplay(value, command, replayed) - const v2Metadata = receiptV2Metadata(value, command) - const result = value.result - const taskIds = value.taskIds - if (value.version !== IMAGE_GENERATION_SCHEMA_VERSION - || value.commandId !== command.intent_id - || value.operation !== command.operation - || value.status !== 'queued' - || !Array.isArray(value.entities) - || !Array.isArray(value.artifacts) - || !Array.isArray(taskIds) - || !Array.isArray(value.pipelineIds) - || !validTaskResult(result, command.input.workspace, taskIds)) { - throw invalidReceipt(command) - } - const receipt: ImageGenerationReceipt = { - version: IMAGE_GENERATION_SCHEMA_VERSION, - commandId: command.intent_id, - operation: IMAGE_GENERATION_OPERATION, - status: 'queued', - entities: JSON.parse(stableSerialize(value.entities)) as unknown[], - artifacts: JSON.parse(stableSerialize(value.artifacts)) as unknown[], - taskIds: [...taskIds] as string[], - pipelineIds: [...value.pipelineIds] as string[], - result: JSON.parse(stableSerialize(result)) as ImageGenerationTaskResult, - ...v2Metadata, - } - if (outerReplayed !== undefined) receipt.replayed = outerReplayed - return receipt -} - -function submissionContextPart(value: unknown, field: string): string | undefined { - if (value === undefined || value === null || value === '') return undefined - if (typeof value !== 'string' || !value.trim() || value.trim() !== value - || value.length > MAX_SUBMISSION_CONTEXT_ID_LENGTH) { - throw new Error(field + ' must be an exact non-blank string of at most 200 characters') - } - return value -} - -function validateSubmissionContext( - context: GenerationSubmissionContext | undefined, -): StoredSubmissionContext | undefined { - if (!context) return undefined - if (!SUBMISSION_ACTORS.has(context.actor)) { - throw new Error('submissionContext.actor must be a known actor') - } - const workflowId = submissionContextPart(context.workflowId, 'submissionContext.workflowId') - const runId = submissionContextPart(context.runId, 'submissionContext.runId') - return { - actor: context.actor, - ...(workflowId !== undefined ? { workflowId } : {}), - ...(runId !== undefined ? { runId } : {}), - } -} - -async function postCommand( - command: ImageGenerationCommand, - submissionContext?: StoredSubmissionContext, -): Promise { - let response: Response - try { - const headers: Record = { 'Content-Type': 'application/json' } - if (submissionContext) { - headers['X-Hocus-UI-Surface'] = submissionContext.actor === 'wizard' ? 'wizard' : 'studio' - const context: Record = {} - const workflowId = submissionContextPart(submissionContext.workflowId, 'submissionContext.workflowId') - const runId = submissionContextPart(submissionContext.runId, 'submissionContext.runId') - if (workflowId !== undefined) context.workflowId = workflowId - if (runId !== undefined) context.runId = runId - if (Object.keys(context).length > 0) headers['X-Hocus-UI-Context'] = JSON.stringify(context) - } - response = await fetch(`${BASE}/api/v1/generation/commands`, { - method: 'POST', - headers, - body: JSON.stringify(command), - }) - } catch (error) { - throw new ImageGenerationCommandError( - error instanceof Error ? error.message : 'Image generation request failed', - command.intent_id, - command.input.workspace, - { uncertain: true, code: 'transport_uncertain' }, - ) - } - if (!response.ok) { - throw await responseError(response, command.intent_id, command.input.workspace) - } - try { - return await response.json() - } catch { - throw new ImageGenerationCommandError( - `Image generation response could not be decoded for ${command.intent_id}`, - command.intent_id, - command.input.workspace, - { status: response.status, uncertain: true, code: 'invalid_response' }, - ) - } -} - -export interface SubmitImageGenerationCommandOptions { - /** - * A declared UI surface is transport metadata, never part of the command - * snapshot and never an authorization decision. - */ - submissionContext?: GenerationSubmissionContext - /** - * Runs after the pending hint is durable and before POST. The callback gets - * a detached copy so it cannot mutate the retry or transport snapshot. - */ - onSnapshotReady?: (snapshot: ImageGenerationCommand) => void | Promise -} - -interface PreparedSubmission { - recovering: boolean - submissionContext?: StoredSubmissionContext -} - -function requestedContext( - snapshot: ImageGenerationCommand, - context: GenerationSubmissionContext | undefined, -): StoredSubmissionContext | undefined { - try { - return validateSubmissionContext(context) - } catch (error) { - throw new ImageGenerationCommandError( - error instanceof Error ? error.message : 'The submission context is invalid', - snapshot.intent_id, - snapshot.input.workspace, - { code: 'invalid_submission_context' }, - ) - } -} - -function assertContextMatches( - snapshot: ImageGenerationCommand, - stored: StoredSubmissionContext | null, - requested: StoredSubmissionContext | undefined, -): void { - if (stored && requested && !sameSubmissionContext(stored, requested)) { - throw new ImageGenerationCommandError( - `intent_id ${snapshot.intent_id} is already attributed to a different UI context`, - snapshot.intent_id, - snapshot.input.workspace, - { code: 'submission_context_conflict', uncertain: true }, - ) - } -} - -function assertPendingCommandMatches( - snapshot: ImageGenerationCommand, - existing: ImageGenerationCommand | null, -): void { - if (existing && !sameCommand(existing, snapshot)) { - throw new ImageGenerationCommandError( - `intent_id ${snapshot.intent_id} is already pending with a different command`, - snapshot.intent_id, - snapshot.input.workspace, - { code: 'intent_conflict' }, - ) - } -} - -function prepareNewPendingContext( - snapshot: ImageGenerationCommand, - requested: StoredSubmissionContext | undefined, -): void { - // A sidecar is written before its command hint. If this write fails, no - // recoverable command is left behind to suggest an admitted request. - // Clearing an orphan is strict for the same reason: a stale attribution - // must not be paired with a newly written command. - persistPendingContext(snapshot, requested) -} - -function persistMissingRecoveryContext( - snapshot: ImageGenerationCommand, - recovering: boolean, - stored: StoredSubmissionContext | null, - requested: StoredSubmissionContext | undefined, -): StoredSubmissionContext | undefined { - const submissionContext = stored || requested - if (recovering && !stored && requested) { - persistPendingContext(snapshot, requested) - } - return submissionContext -} - -function preparePendingSubmission( - snapshot: ImageGenerationCommand, - requested: StoredSubmissionContext | undefined, -): PreparedSubmission { - let recovering = false - try { - const existing = readPending(snapshot.intent_id) - assertPendingCommandMatches(snapshot, existing) - recovering = existing !== null - if (!recovering) prepareNewPendingContext(snapshot, requested) - recovering = retainPending(snapshot) - const stored = recovering ? readPendingContext(snapshot) : null - assertContextMatches(snapshot, stored, requested) - return { - recovering, - submissionContext: persistMissingRecoveryContext(snapshot, recovering, stored, requested), - } - } catch (error) { - if (error instanceof ImageGenerationCommandError) throw error - throw new ImageGenerationCommandError( - error instanceof Error ? error.message : 'Could not persist image generation command', - snapshot.intent_id, - snapshot.input.workspace, - { code: 'pending_storage_failed', uncertain: recovering }, - ) - } -} - -async function presentSnapshot( - snapshot: ImageGenerationCommand, - recovering: boolean, - hook: SubmitImageGenerationCommandOptions['onSnapshotReady'], -): Promise { - if (!hook) return - try { - await hook(detachedCommand(snapshot)) - } catch (error) { - // A hook failure happens before network admission and is therefore - // certain. Preserve an older recovery hint because it may represent a - // previously admitted request whose response was lost. - if (!recovering) forgetPending(snapshot) - throw new ImageGenerationCommandError( - error instanceof Error ? error.message : 'The image command snapshot could not be presented', - snapshot.intent_id, - snapshot.input.workspace, - { code: 'snapshot_hook_failed' }, - ) - } -} - -async function admitImageCommand( - snapshot: ImageGenerationCommand, - submissionContext: StoredSubmissionContext | undefined, -): Promise { - const envelope = unwrapReceipt(await postCommand(snapshot, submissionContext)) - if (envelope.malformed) throw invalidReceipt(snapshot) - const receipt = validateReceipt(envelope.receipt, snapshot, envelope.replayed) - // A committed receipt is returned even if best-effort local cleanup fails. - forgetPending(snapshot) - return receipt -} - -function isDefinitiveClientError(error: ImageGenerationCommandError): boolean { - return error.status !== undefined && error.status >= 400 && error.status < 500 -} - -function normalizeSubmissionFailure( - error: unknown, - snapshot: ImageGenerationCommand, - recovering: boolean, -): ImageGenerationCommandError { - const commandError = error instanceof ImageGenerationCommandError - ? error - : new ImageGenerationCommandError( - error instanceof Error ? error.message : 'Image generation command failed', - snapshot.intent_id, - snapshot.input.workspace, - { uncertain: true, code: 'unknown_failure' }, - ) - // A first, explicit 4xx response is definitive before admission. Once a - // pending hint exists, a later rejection may follow an admitted request; - // preserve it, including a 401 after a timeout or lost response. - if (!recovering && isDefinitiveClientError(commandError)) forgetPending(snapshot) - return new ImageGenerationCommandError( - commandError.message, - snapshot.intent_id, - snapshot.input.workspace, - { - status: commandError.status, - uncertain: commandError.uncertain || recovering, - code: commandError.code, - }, - ) -} - -/** Submit or explicitly retry the same detached envelope and intention. */ -export async function submitImageGenerationCommand( - command: ImageGenerationCommand, - options: SubmitImageGenerationCommandOptions = {}, -): Promise { - const snapshot = detachedCommand(command) - const requestedSubmissionContext = requestedContext(snapshot, options.submissionContext) - const prepared = preparePendingSubmission(snapshot, requestedSubmissionContext) - - await presentSnapshot(snapshot, prepared.recovering, options.onSnapshotReady) - try { - return await admitImageCommand(snapshot, prepared.submissionContext) - } catch (error) { - throw normalizeSubmissionFailure(error, snapshot, prepared.recovering) - } -} - -function receiptCommandContext(intentId: string, workspace: string): ReceiptContext { - const fallback: ReceiptContext = { - version: 1, - intent_id: intentId, - operation: IMAGE_GENERATION_OPERATION, - input: { workspace }, - } - try { - const pending = readPending(intentId) - return pending && pending.input.workspace === workspace ? pending : fallback - } catch { - // A receipt read must remain available when local recovery storage is damaged. - return fallback - } -} - -async function requestReceipt(workspace: string, intentId: string): Promise { - try { - const response = await fetch( - `${BASE}/api/v1/generation/commands/receipt?workspace=${encodeURIComponent(workspace)}&intent_id=${encodeURIComponent(intentId)}`, - ) - if (!response.ok) throw await responseError(response, intentId, workspace) - return response - } catch (error) { - if (error instanceof ImageGenerationCommandError) throw error - throw new ImageGenerationCommandError( - error instanceof Error ? error.message : 'Receipt request failed', - intentId, - workspace, - { uncertain: true, code: 'transport_uncertain' }, - ) - } -} - -async function decodeReceiptResponse( - response: Response, - command: ReceiptContext, -): Promise { - try { - const payload = unwrapReceipt(await response.json()) - if (payload.malformed) throw invalidReceipt(command) - return validateReceipt(payload.receipt, command, payload.replayed) - } catch (error) { - if (error instanceof ImageGenerationCommandError) throw error - throw new ImageGenerationCommandError( - error instanceof Error ? error.message : `Receipt could not be verified for ${command.intent_id}`, - command.intent_id, - command.input.workspace, - { status: 200, uncertain: true, code: 'invalid_receipt' }, - ) - } -} - -function clearReceiptPending(intentId: string, workspace: string): void { - try { - const pending = readPending(intentId) - if (pending && pending.input.workspace === workspace) clearPendingReceipt(pending) - } catch { /* Keep the valid receipt even if local recovery storage is corrupt. */ } -} - -function clearPendingReceipt(command: ImageGenerationCommand): void { - try { retainPending(command, true) } catch { /* Keep the receipt visible if storage cleanup is unavailable. */ } -} - -/** Query a durable receipt after a lost response; this never invents a new ID. */ -export async function fetchImageGenerationCommandReceipt( - workspace: string, - intentId: string, -): Promise { - requiredText(workspace, 'workspace', MAX_ID_LENGTH) - requiredText(intentId, 'intent_id', MAX_INTENT_LENGTH) - // A receipt query has no command envelope of its own. When the durable hint - // is present, use its version so a lost v2 response cannot be accepted as a - // legacy v1 receipt. If the hint is unavailable, the endpoint remains a - // legacy-compatible read and the server's receipt metadata is still - // validated when it is present. - const receiptCommand = receiptCommandContext(intentId, workspace) - const response = await requestReceipt(workspace, intentId) - const receipt = await decodeReceiptResponse(response, receiptCommand) - // Receipt validation is authoritative. Storage read/removal is only a - // recovery hint and must never turn a valid GET into an apparent failure. - clearReceiptPending(intentId, workspace) - return receipt -} +export type SubmitImageGenerationCommandOptions = SubmitGenerationCommandOptions +export const submitImageGenerationCommand = imageCommandClient.submit +export const fetchImageGenerationCommandReceipt = imageCommandClient.fetchReceipt export const getImageGenerationCommandReceipt = fetchImageGenerationCommandReceipt export function subscribeImageGenerationCommands(callback: () => void): () => void { - window.addEventListener(PENDING_CHANGED_EVENT, callback) - window.addEventListener('storage', callback) - return () => { - window.removeEventListener(PENDING_CHANGED_EVENT, callback) - window.removeEventListener('storage', callback) - } + return imageCommandClient.subscribe(callback) } diff --git a/ui/src/api/speechCommandCatalog.json b/ui/src/api/speechCommandCatalog.json new file mode 100644 index 000000000..146d058fd --- /dev/null +++ b/ui/src/api/speechCommandCatalog.json @@ -0,0 +1,1973 @@ +{ + "version": 2, + "operations": [ + { + "name": "generation.speech", + "version": 2, + "supportedVersions": [ + 2 + ], + "domain": "studio", + "mutation": true, + "description": "Admit speech using an installed speech model, literal text, voice settings and canonical audio references in an explicit output workspace. Preserve the original speaker text separately from the effective native prompt. Reuse intent_id only for retries; the receipt proves admission, and its task reports completion.", + "inputSchema": { + "type": "object", + "additionalProperties": false, + "$defs": { + "StudioSpeechCustomSettings": { + "additionalProperties": false, + "description": "Typed union of custom settings exposed by speech handlers.\n\nThe selected model still owns which subset is valid and its metadata\nranges. Keeping the known IDs closed here prevents arbitrary nested JSON\nfrom crossing the command boundary while allowing each speech handler's\ncurrently published setting family.", + "properties": { + "auto_split_every_s": { + "anyOf": [ + { + "type": "number" + }, + { + "const": "", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Auto Split Every S" + }, + "exaggeration": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Exaggeration" + }, + "pace": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Pace" + }, + "vc_steps": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Vc Steps" + }, + "vc_cfg_rate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Vc Cfg Rate" + }, + "duration_multiplier": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Multiplier" + } + }, + "title": "StudioSpeechCustomSettings", + "type": "object" + }, + "StudioSpeechParams": { + "additionalProperties": false, + "description": "Typed native speech parameters emitted by Studio.", + "properties": { + "prompt": { + "maxLength": 200000, + "minLength": 1, + "title": "Prompt", + "type": "string" + }, + "alt_prompt": { + "default": "", + "maxLength": 200000, + "title": "Alt Prompt", + "type": "string" + }, + "model_type": { + "maxLength": 240, + "minLength": 1, + "title": "Model Type", + "type": "string" + }, + "resolution": { + "maxLength": 128, + "minLength": 1, + "title": "Resolution", + "type": "string" + }, + "video_length": { + "default": 0, + "maximum": 0, + "minimum": 0, + "title": "Video Length", + "type": "integer" + }, + "num_inference_steps": { + "default": 0, + "maximum": 100000, + "minimum": 0, + "title": "Num Inference Steps", + "type": "integer" + }, + "guidance_scale": { + "default": 1.0, + "maximum": 1000.0, + "minimum": 0.0, + "title": "Guidance Scale", + "type": "number" + }, + "seed": { + "default": -1, + "maximum": 9223372036854775807, + "minimum": -9223372036854775808, + "title": "Seed", + "type": "integer" + }, + "image_mode": { + "default": 0, + "maximum": 0, + "minimum": 0, + "title": "Image Mode", + "type": "integer" + }, + "generation_mode": { + "const": "audio", + "default": "audio", + "title": "Generation Mode", + "type": "string" + }, + "negative_prompt": { + "default": "", + "maxLength": 200000, + "title": "Negative Prompt", + "type": "string" + }, + "repeat_generation": { + "default": 1, + "maximum": 100, + "minimum": 1, + "title": "Repeat Generation", + "type": "integer" + }, + "activated_loras": { + "items": { + "maxLength": 8192, + "minLength": 1, + "type": "string" + }, + "maxItems": 64, + "title": "Activated Loras", + "type": "array" + }, + "loras_multipliers": { + "default": "", + "maxLength": 8192, + "title": "Loras Multipliers", + "type": "string" + }, + "multi_prompts_gen_type": { + "default": 2, + "maximum": 2, + "minimum": 2, + "title": "Multi Prompts Gen Type", + "type": "integer" + }, + "audio_prompt_type": { + "default": "", + "maxLength": 32, + "pattern": "^[A-Za-z0-9]*$", + "title": "Audio Prompt Type", + "type": "string" + }, + "audio_guide": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide" + }, + "audio_guide2": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide2" + }, + "audio_guide3": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide3" + }, + "audio_guide4": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide4" + }, + "audio_guide5": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide5" + }, + "audio_guide6": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide6" + }, + "audio_source": { + "default": null, + "enum": [ + "", + null + ], + "title": "Audio Source" + }, + "image_start": { + "anyOf": [ + { + "enum": [ + "", + null + ] + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], + "default": null, + "title": "Image Start" + }, + "image_end": { + "anyOf": [ + { + "enum": [ + "", + null + ] + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], + "default": null, + "title": "Image End" + }, + "image_refs": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Image Refs" + }, + "image_guide": { + "anyOf": [ + { + "enum": [ + "", + null + ] + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], + "default": null, + "title": "Image Guide" + }, + "image_mask": { + "anyOf": [ + { + "enum": [ + "", + null + ] + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], + "default": null, + "title": "Image Mask" + }, + "video_guide": { + "default": null, + "enum": [ + "", + null + ], + "title": "Video Guide" + }, + "video_mask": { + "default": null, + "enum": [ + "", + null + ], + "title": "Video Mask" + }, + "video_source": { + "default": null, + "enum": [ + "", + null + ], + "title": "Video Source" + }, + "spatial_upsampling": { + "default": "", + "enum": [ + "", + null + ], + "title": "Spatial Upsampling" + }, + "temporal_upsampling": { + "default": "", + "enum": [ + "", + null + ], + "title": "Temporal Upsampling" + }, + "wangp_processor_settings": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Wangp Processor Settings" + }, + "MMAudio_setting": { + "anyOf": [ + { + "maximum": 0, + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Mmaudio Setting" + }, + "MMAudio_prompt": { + "default": null, + "enum": [ + "", + null + ], + "title": "Mmaudio Prompt" + }, + "MMAudio_neg_prompt": { + "default": null, + "enum": [ + "", + null + ], + "title": "Mmaudio Neg Prompt" + }, + "h3_ref_videos": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "H3 Ref Videos" + }, + "h3_ref_audios": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "H3 Ref Audios" + }, + "minimax_h3_references": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Minimax H3 References" + }, + "_audio_sub_mode": { + "const": "speech", + "default": "speech", + "title": "Audio Sub Mode", + "type": "string" + }, + "_tts_original_prompt": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Original Prompt" + }, + "_tts_speaker_name1": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name1" + }, + "_tts_speaker_name2": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name2" + }, + "_tts_speaker_name3": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name3" + }, + "_tts_speaker_name4": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name4" + }, + "_tts_speaker_name5": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name5" + }, + "_tts_speaker_name6": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name6" + }, + "_tts_voice_count": { + "default": 0, + "maximum": 6, + "minimum": 0, + "title": "Tts Voice Count", + "type": "integer" + }, + "duration_seconds": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Seconds" + }, + "pause_seconds": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Pause Seconds" + }, + "temperature": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Temperature" + }, + "top_p": { + "anyOf": [ + { + "maximum": 1.0, + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Top P" + }, + "top_k": { + "anyOf": [ + { + "maximum": 100000, + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Top K" + }, + "audio_scale": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Scale" + }, + "audio_guidance_scale": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guidance Scale" + }, + "alt_scale": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Alt Scale" + }, + "guidance_phases": { + "default": 1, + "maximum": 16, + "minimum": 0, + "title": "Guidance Phases", + "type": "integer" + }, + "flow_shift": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Flow Shift" + }, + "sample_solver": { + "default": "", + "maxLength": 8192, + "title": "Sample Solver", + "type": "string" + }, + "settings_version": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Settings Version" + }, + "prompt_enhancer": { + "default": "", + "enum": [ + "", + null + ], + "title": "Prompt Enhancer" + }, + "model_mode": { + "anyOf": [ + { + "maxLength": 256, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Mode" + }, + "custom_settings": { + "anyOf": [ + { + "$ref": "#/$defs/StudioSpeechCustomSettings" + }, + { + "type": "null" + } + ], + "default": null + }, + "tts_dynaudnorm": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "integer" + }, + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Dynaudnorm" + }, + "tts_comp_threshold": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Comp Threshold" + }, + "tts_comp_attack": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Comp Attack" + }, + "tts_comp_release": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Comp Release" + }, + "tts_comp_makeup": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Comp Makeup" + }, + "minimax_h3_turbo_mode": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Minimax H3 Turbo Mode" + } + }, + "required": [ + "prompt", + "model_type", + "resolution" + ], + "title": "StudioSpeechParams", + "type": "object" + } + }, + "properties": { + "version": { + "type": "integer", + "const": 2 + }, + "operation": { + "const": "generation.speech" + }, + "intent_id": { + "maxLength": 160, + "minLength": 1, + "title": "Intent Id", + "type": "string" + }, + "input": { + "additionalProperties": false, + "properties": { + "workspace": { + "maxLength": 240, + "minLength": 1, + "pattern": "^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$", + "title": "Workspace", + "type": "string" + }, + "workspace_collection_id": { + "anyOf": [ + { + "maxLength": 200, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Workspace Collection Id" + }, + "params": { + "$ref": "#/$defs/StudioSpeechParams" + } + }, + "required": [ + "workspace", + "params" + ], + "title": "StudioSpeechInput", + "type": "object" + } + }, + "required": [ + "version", + "operation", + "intent_id", + "input" + ] + } + } + ], + "studio": { + "version": 2, + "operation": "generation.speech", + "intent_id": { + "maxLength": 160, + "minLength": 1, + "title": "Intent Id", + "type": "string" + }, + "input": { + "$defs": { + "StudioSpeechCustomSettings": { + "additionalProperties": false, + "description": "Typed union of custom settings exposed by speech handlers.\n\nThe selected model still owns which subset is valid and its metadata\nranges. Keeping the known IDs closed here prevents arbitrary nested JSON\nfrom crossing the command boundary while allowing each speech handler's\ncurrently published setting family.", + "properties": { + "auto_split_every_s": { + "anyOf": [ + { + "type": "number" + }, + { + "const": "", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Auto Split Every S" + }, + "exaggeration": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Exaggeration" + }, + "pace": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Pace" + }, + "vc_steps": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Vc Steps" + }, + "vc_cfg_rate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Vc Cfg Rate" + }, + "duration_multiplier": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Multiplier" + } + }, + "title": "StudioSpeechCustomSettings", + "type": "object" + }, + "StudioSpeechParams": { + "additionalProperties": false, + "description": "Typed native speech parameters emitted by Studio.", + "properties": { + "prompt": { + "maxLength": 200000, + "minLength": 1, + "title": "Prompt", + "type": "string" + }, + "alt_prompt": { + "default": "", + "maxLength": 200000, + "title": "Alt Prompt", + "type": "string" + }, + "model_type": { + "maxLength": 240, + "minLength": 1, + "title": "Model Type", + "type": "string" + }, + "resolution": { + "maxLength": 128, + "minLength": 1, + "title": "Resolution", + "type": "string" + }, + "video_length": { + "default": 0, + "maximum": 0, + "minimum": 0, + "title": "Video Length", + "type": "integer" + }, + "num_inference_steps": { + "default": 0, + "maximum": 100000, + "minimum": 0, + "title": "Num Inference Steps", + "type": "integer" + }, + "guidance_scale": { + "default": 1.0, + "maximum": 1000.0, + "minimum": 0.0, + "title": "Guidance Scale", + "type": "number" + }, + "seed": { + "default": -1, + "maximum": 9223372036854775807, + "minimum": -9223372036854775808, + "title": "Seed", + "type": "integer" + }, + "image_mode": { + "default": 0, + "maximum": 0, + "minimum": 0, + "title": "Image Mode", + "type": "integer" + }, + "generation_mode": { + "const": "audio", + "default": "audio", + "title": "Generation Mode", + "type": "string" + }, + "negative_prompt": { + "default": "", + "maxLength": 200000, + "title": "Negative Prompt", + "type": "string" + }, + "repeat_generation": { + "default": 1, + "maximum": 100, + "minimum": 1, + "title": "Repeat Generation", + "type": "integer" + }, + "activated_loras": { + "items": { + "maxLength": 8192, + "minLength": 1, + "type": "string" + }, + "maxItems": 64, + "title": "Activated Loras", + "type": "array" + }, + "loras_multipliers": { + "default": "", + "maxLength": 8192, + "title": "Loras Multipliers", + "type": "string" + }, + "multi_prompts_gen_type": { + "default": 2, + "maximum": 2, + "minimum": 2, + "title": "Multi Prompts Gen Type", + "type": "integer" + }, + "audio_prompt_type": { + "default": "", + "maxLength": 32, + "pattern": "^[A-Za-z0-9]*$", + "title": "Audio Prompt Type", + "type": "string" + }, + "audio_guide": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide" + }, + "audio_guide2": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide2" + }, + "audio_guide3": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide3" + }, + "audio_guide4": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide4" + }, + "audio_guide5": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide5" + }, + "audio_guide6": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guide6" + }, + "audio_source": { + "default": null, + "enum": [ + "", + null + ], + "title": "Audio Source" + }, + "image_start": { + "anyOf": [ + { + "enum": [ + "", + null + ] + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], + "default": null, + "title": "Image Start" + }, + "image_end": { + "anyOf": [ + { + "enum": [ + "", + null + ] + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], + "default": null, + "title": "Image End" + }, + "image_refs": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Image Refs" + }, + "image_guide": { + "anyOf": [ + { + "enum": [ + "", + null + ] + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], + "default": null, + "title": "Image Guide" + }, + "image_mask": { + "anyOf": [ + { + "enum": [ + "", + null + ] + }, + { + "items": { + "type": "string" + }, + "type": "array" + } + ], + "default": null, + "title": "Image Mask" + }, + "video_guide": { + "default": null, + "enum": [ + "", + null + ], + "title": "Video Guide" + }, + "video_mask": { + "default": null, + "enum": [ + "", + null + ], + "title": "Video Mask" + }, + "video_source": { + "default": null, + "enum": [ + "", + null + ], + "title": "Video Source" + }, + "spatial_upsampling": { + "default": "", + "enum": [ + "", + null + ], + "title": "Spatial Upsampling" + }, + "temporal_upsampling": { + "default": "", + "enum": [ + "", + null + ], + "title": "Temporal Upsampling" + }, + "wangp_processor_settings": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Wangp Processor Settings" + }, + "MMAudio_setting": { + "anyOf": [ + { + "maximum": 0, + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Mmaudio Setting" + }, + "MMAudio_prompt": { + "default": null, + "enum": [ + "", + null + ], + "title": "Mmaudio Prompt" + }, + "MMAudio_neg_prompt": { + "default": null, + "enum": [ + "", + null + ], + "title": "Mmaudio Neg Prompt" + }, + "h3_ref_videos": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "H3 Ref Videos" + }, + "h3_ref_audios": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "H3 Ref Audios" + }, + "minimax_h3_references": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Minimax H3 References" + }, + "_audio_sub_mode": { + "const": "speech", + "default": "speech", + "title": "Audio Sub Mode", + "type": "string" + }, + "_tts_original_prompt": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Original Prompt" + }, + "_tts_speaker_name1": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name1" + }, + "_tts_speaker_name2": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name2" + }, + "_tts_speaker_name3": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name3" + }, + "_tts_speaker_name4": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name4" + }, + "_tts_speaker_name5": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name5" + }, + "_tts_speaker_name6": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": "", + "title": "Tts Speaker Name6" + }, + "_tts_voice_count": { + "default": 0, + "maximum": 6, + "minimum": 0, + "title": "Tts Voice Count", + "type": "integer" + }, + "duration_seconds": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Duration Seconds" + }, + "pause_seconds": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Pause Seconds" + }, + "temperature": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Temperature" + }, + "top_p": { + "anyOf": [ + { + "maximum": 1.0, + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Top P" + }, + "top_k": { + "anyOf": [ + { + "maximum": 100000, + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Top K" + }, + "audio_scale": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Scale" + }, + "audio_guidance_scale": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Audio Guidance Scale" + }, + "alt_scale": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Alt Scale" + }, + "guidance_phases": { + "default": 1, + "maximum": 16, + "minimum": 0, + "title": "Guidance Phases", + "type": "integer" + }, + "flow_shift": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Flow Shift" + }, + "sample_solver": { + "default": "", + "maxLength": 8192, + "title": "Sample Solver", + "type": "string" + }, + "settings_version": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Settings Version" + }, + "prompt_enhancer": { + "default": "", + "enum": [ + "", + null + ], + "title": "Prompt Enhancer" + }, + "model_mode": { + "anyOf": [ + { + "maxLength": 256, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Mode" + }, + "custom_settings": { + "anyOf": [ + { + "$ref": "#/$defs/StudioSpeechCustomSettings" + }, + { + "type": "null" + } + ], + "default": null + }, + "tts_dynaudnorm": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "integer" + }, + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Dynaudnorm" + }, + "tts_comp_threshold": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Comp Threshold" + }, + "tts_comp_attack": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Comp Attack" + }, + "tts_comp_release": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Comp Release" + }, + "tts_comp_makeup": { + "anyOf": [ + { + "minimum": 0.0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tts Comp Makeup" + }, + "minimax_h3_turbo_mode": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Minimax H3 Turbo Mode" + } + }, + "required": [ + "prompt", + "model_type", + "resolution" + ], + "title": "StudioSpeechParams", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "workspace": { + "maxLength": 240, + "minLength": 1, + "pattern": "^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$", + "title": "Workspace", + "type": "string" + }, + "workspace_collection_id": { + "anyOf": [ + { + "maxLength": 200, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Workspace Collection Id" + }, + "params": { + "$ref": "#/$defs/StudioSpeechParams" + } + }, + "required": [ + "workspace", + "params" + ], + "title": "StudioSpeechInput", + "type": "object" + }, + "supported_input_fields": [ + "prompt", + "alt_prompt", + "model_type", + "resolution", + "video_length", + "num_inference_steps", + "guidance_scale", + "seed", + "image_mode", + "generation_mode", + "negative_prompt", + "repeat_generation", + "activated_loras", + "loras_multipliers", + "audio_prompt_type", + "audio_guide", + "audio_guide2", + "audio_guide3", + "audio_guide4", + "audio_guide5", + "audio_guide6", + "audio_source", + "_audio_sub_mode", + "_tts_original_prompt", + "_tts_speaker_name1", + "_tts_speaker_name2", + "_tts_speaker_name3", + "_tts_speaker_name4", + "_tts_speaker_name5", + "_tts_speaker_name6", + "_tts_voice_count", + "duration_seconds", + "pause_seconds", + "temperature", + "top_p", + "top_k", + "audio_scale", + "audio_guidance_scale", + "alt_scale", + "guidance_phases", + "flow_shift", + "sample_solver", + "settings_version", + "prompt_enhancer", + "model_mode", + "custom_settings", + "tts_dynaudnorm", + "tts_comp_threshold", + "tts_comp_attack", + "tts_comp_release", + "tts_comp_makeup", + "multi_prompts_gen_type", + "minimax_h3_turbo_mode", + "image_start", + "image_end", + "image_refs", + "image_guide", + "image_mask", + "video_guide", + "video_mask", + "video_source", + "spatial_upsampling", + "temporal_upsampling", + "wangp_processor_settings", + "MMAudio_setting", + "MMAudio_prompt", + "MMAudio_neg_prompt", + "h3_ref_videos", + "h3_ref_audios", + "minimax_h3_references" + ], + "speech_model_types": [ + "chatterbox", + "dramabox_audio", + "index_tts2", + "kugelaudio_0_open", + "qwen3_tts_base", + "qwen3_tts_customvoice", + "qwen3_tts_voicedesign", + "scenema_audio" + ], + "effects": { + "generation_mode": "audio", + "_audio_sub_mode": "speech", + "video_length": 0, + "image_mode": 0, + "multi_prompts_gen_type": 2, + "negative_prompt": "", + "repeat_generation": 1, + "activated_loras": [], + "loras_multipliers": "", + "audio_prompt_type": "", + "prompt_enhancer": "", + "minimax_h3_turbo_mode": false, + "_tts_speaker_name1": "", + "_tts_speaker_name2": "", + "_tts_speaker_name3": "", + "_tts_speaker_name4": "", + "_tts_speaker_name5": "", + "_tts_speaker_name6": "", + "_tts_voice_count": 0 + }, + "inactive": [ + "image_mode", + "video_length", + "minimax_h3_turbo_mode", + "image_start", + "image_end", + "image_refs", + "image_guide", + "image_mask", + "video_guide", + "video_mask", + "video_source", + "audio_source", + "MMAudio_setting", + "MMAudio_prompt", + "MMAudio_neg_prompt", + "h3_ref_videos", + "h3_ref_audios", + "minimax_h3_references", + "spatial_upsampling", + "temporal_upsampling", + "wangp_processor_settings" + ], + "excluded": [ + "actor", + "client", + "permission", + "provenance", + "workspace inside input.params", + "filesystem paths", + "free-form provider payloads", + "music and SFX model types", + "video, avatar and model3d controls" + ] + } +} diff --git a/ui/src/api/speechGenerationCommands.ts b/ui/src/api/speechGenerationCommands.ts new file mode 100644 index 000000000..d17dc8490 --- /dev/null +++ b/ui/src/api/speechGenerationCommands.ts @@ -0,0 +1,92 @@ +import { + createGenerationCommandClient, + GenerationCommandError, + type GenerationReceiptLike, + type GenerationTaskResult, + type GenerationCommandErrorOptions, + type SubmitGenerationCommandOptions, +} from './generationCommandClient' +import { + assertStudioSpeechGenerationCommand, + buildStudioSpeechGenerationCommand, + createStudioSpeechGenerationCommand, + detachedStudioSpeechGenerationCommand, + type StudioSpeechGenerationCommand, + type StudioSpeechGenerationFullParams, + type StudioSpeechGenerationInput, + type StudioSpeechParamKey, + type StudioSpeechParams, + STUDIO_SPEECH_OPERATION, + STUDIO_SPEECH_SCHEMA_VERSION, +} from '../features/studio/speechGenerationSpec' + +export { + assertStudioSpeechGenerationCommand, + buildStudioSpeechGenerationCommand, + createStudioSpeechGenerationCommand, + detachedStudioSpeechGenerationCommand, + STUDIO_SPEECH_OPERATION, + STUDIO_SPEECH_SCHEMA_VERSION, +} +export type { + StudioSpeechGenerationCommand, + StudioSpeechGenerationFullParams, + StudioSpeechGenerationInput, + StudioSpeechParamKey, + StudioSpeechParams, +} + +export type SpeechGenerationTaskResult = GenerationTaskResult + +export interface SpeechGenerationReceipt extends GenerationReceiptLike { + version: 1 + operation: typeof STUDIO_SPEECH_OPERATION + result: SpeechGenerationTaskResult +} + +export class SpeechGenerationCommandError extends GenerationCommandError { + constructor( + message: string, + intentId: string, + workspace: string, + options: GenerationCommandErrorOptions = {}, + ) { + super(message, intentId, workspace, { + ...options, + code: options.code ?? 'speech_generation_command_failed', + }) + this.name = 'SpeechGenerationCommandError' + } +} + +const speechCommandClient = createGenerationCommandClient({ + storagePrefix: 'hocuspocus.generation.speech-commands.v2:', + contextStoragePrefix: 'hocuspocus.generation.speech-command-context.v1:', + pendingChangedEvent: 'hocuspocus:generation-speech-commands-changed', + operation: STUDIO_SPEECH_OPERATION, + label: 'Speech generation', + receiptFallbackVersion: STUDIO_SPEECH_SCHEMA_VERSION, + detach: detachedStudioSpeechGenerationCommand, + errorClass: SpeechGenerationCommandError, + castReceipt: value => value as SpeechGenerationReceipt, +}) + +export const newSpeechGenerationIntentId = speechCommandClient.newIntentId +export const pendingSpeechGenerationCommands = speechCommandClient.pendingCommands +export const pendingSpeechGenerationCommand = speechCommandClient.pendingCommand + +export type SubmitSpeechGenerationCommandOptions = SubmitGenerationCommandOptions + +export const submitSpeechGenerationCommand = speechCommandClient.submit +export const fetchSpeechGenerationCommandReceipt = speechCommandClient.fetchReceipt +export const getSpeechGenerationCommandReceipt = fetchSpeechGenerationCommandReceipt + +export function subscribeSpeechGenerationCommands(callback: () => void): () => void { + return speechCommandClient.subscribe(callback) +} + +/** + * Symmetric name for callers that already distinguish legacy Speech from the + * typed Studio envelope. The builder still receives the complete native map. + */ +export const createSpeechGenerationCommand = createStudioSpeechGenerationCommand diff --git a/ui/src/components/Sidebar/Sidebar.tsx b/ui/src/components/Sidebar/Sidebar.tsx index 8ae61ab6a..eb5c740ae 100644 --- a/ui/src/components/Sidebar/Sidebar.tsx +++ b/ui/src/components/Sidebar/Sidebar.tsx @@ -37,9 +37,9 @@ import { PanoramaLoopPanel } from './PanoramaLoopPanel' import { BrandIdentity } from '../BrandIdentity' import { DirectorChat } from './DirectorChat' import { useUiTranslation } from '../../i18n' +import { StudioCommandPanels } from '../../features/studio/StudioCommandPanels' const ViggleControls = lazy(() => import('./ViggleControls').then(module => ({ default: module.ViggleControls }))) -const StudioImageCommandPanel = lazy(() => import('../../features/studio/StudioImageCommandPanel').then(module => ({ default: module.StudioImageCommandPanel }))) export function Sidebar() { const { t } = useUiTranslation('navigation') @@ -104,8 +104,17 @@ export function Sidebar() { window.localStorage.setItem('hocuspocus-tools-sidebar-collapsed', 'false') setSidebarOpen(true) } + const openSpeechSubmission = () => { + setToolsCollapsed(false) + window.localStorage.setItem('hocuspocus-tools-sidebar-collapsed', 'false') + setSidebarOpen(true) + } window.addEventListener('hocuspocus:studio-image-open', openImageSubmission) - return () => window.removeEventListener('hocuspocus:studio-image-open', openImageSubmission) + window.addEventListener('hocuspocus:studio-speech-open', openSpeechSubmission) + return () => { + window.removeEventListener('hocuspocus:studio-image-open', openImageSubmission) + window.removeEventListener('hocuspocus:studio-speech-open', openSpeechSubmission) + } }, [setSidebarOpen]) useEffect(() => { @@ -248,13 +257,9 @@ export function Sidebar() { {/* Prompt area (non-edit modes, skip for SFX/Mixer/Music which have their own UI) */} {!isEdit && !(isAudio && (audioSubMode === 'sfx' || audioSubMode === 'mixer' || audioSubMode === 'music')) && (isMultiClip ? : )} - {isImage && { - await useStore.getState().reconnectJobs() - if (useStore.getState().activeWorkspace === receipt.result.workspace) { - await useStore.getState().maybeRefreshGallery() - } - }} />} + {/* Video: reference images below prompt. In Frames mode the InputsPanel renders them as ordered tiles instead. */} diff --git a/ui/src/features/agent/studioCapabilities.ts b/ui/src/features/agent/studioCapabilities.ts index 1b981f411..827c5fb70 100644 --- a/ui/src/features/agent/studioCapabilities.ts +++ b/ui/src/features/agent/studioCapabilities.ts @@ -118,16 +118,24 @@ function imageAction(raw: Record): AgentPrepareImageAction | nu } function audioAction(raw: Record): AgentPrepareAudioAction | null { - const prompt = text(raw.prompt, 8_000) - if (!prompt) return null const subMode = text(raw.audio_sub_mode, 12) as AgentPrepareAudioAction['subMode'] + const speech = subMode === 'speech' + // Speech text is authored content. Keep every character, including + // whitespace and newlines, and let the closed speech command schema reject + // anything beyond its native 200k limit instead of silently truncating it. + const prompt = speech ? literalText(raw.prompt, 200_000) : text(raw.prompt, 8_000) + if (!prompt?.trim()) return null + const negativePrompt = speech + ? (raw.negative_prompt === undefined ? undefined : literalText(raw.negative_prompt, 200_000)) + : text(raw.negative_prompt, 2_000) || undefined + if (raw.negative_prompt !== undefined && negativePrompt === undefined) return null return { type: 'prepare_audio', subMode: AUDIO_SUB_MODES.has(subMode) ? subMode : 'sfx', prompt, modelType: text(raw.model_type, 160) || undefined, - durationSeconds: number(raw.duration_seconds, 1, 20), - negativePrompt: text(raw.negative_prompt, 2_000) || undefined, + durationSeconds: number(raw.duration_seconds, speech ? 0 : 1, speech ? 1_800 : 20), + negativePrompt, } } @@ -262,7 +270,12 @@ export function registerStudioCapabilities(register: typeof defineCapability): v resolve: audioAction, validate(action) { return action.prompt ? validType('prepare_audio', action) : ['prompt is required'] }, async prepare(action) { - return compilePromptAction(action, action.subMode === 'speech' ? 'speech' : action.subMode === 'music' ? 'music' : 'sfx') + // Language intent remains workflow metadata. The speech native request + // carries authored text exactly; provider-side language handling belongs + // to its model/preflight contract, not this capability parser. + return action.subMode === 'speech' + ? action + : compilePromptAction(action, action.subMode === 'music' ? 'music' : 'sfx') }, async execute(action, context) { return context.adapters.studio.prepareAudio(action) }, correlate(_action, outcome) { return outcome.target }, async track(_action, outcome) { return outcome }, diff --git a/ui/src/features/studio/StudioCommandPanels.tsx b/ui/src/features/studio/StudioCommandPanels.tsx new file mode 100644 index 000000000..019cc12b4 --- /dev/null +++ b/ui/src/features/studio/StudioCommandPanels.tsx @@ -0,0 +1,30 @@ +import { lazy, Suspense } from 'react' +import type { GenerationReceiptLike } from '../../api/generationCommandClient' +import { useStore } from '../../stores/useStore' + +const ImagePanel = lazy(() => import('./StudioImageCommandPanel').then(module => ({ default: module.StudioImageCommandPanel }))) +const SpeechPanel = lazy(() => import('./StudioSpeechCommandPanel').then(module => ({ default: module.StudioSpeechCommandPanel }))) + +async function reconnect(receipt: GenerationReceiptLike): Promise { + await useStore.getState().reconnectJobs() + if (useStore.getState().activeWorkspace === receipt.result.workspace) { + await useStore.getState().maybeRefreshGallery() + } +} + +interface Props { + mode: string + audioSubMode: string + workspace: string + model: string + visible: boolean +} + +/** Only load the durable presentation for the selected Studio operation. */ +export function StudioCommandPanels({ mode, audioSubMode, workspace, model, visible }: Props) { + const Panel = mode === 'image' ? ImagePanel : mode === 'audio' && audioSubMode === 'speech' ? SpeechPanel : null + if (!Panel) return null + return + + +} diff --git a/ui/src/features/studio/StudioSpeechCommandPanel.tsx b/ui/src/features/studio/StudioSpeechCommandPanel.tsx new file mode 100644 index 000000000..e9d1e26b5 --- /dev/null +++ b/ui/src/features/studio/StudioSpeechCommandPanel.tsx @@ -0,0 +1,173 @@ +import { useEffect, useLayoutEffect, useRef, useState } from 'react' +import type { SpeechGenerationReceipt } from '../../api/speechGenerationCommands' +import { + pendingSpeechGenerationCommands, + submitSpeechGenerationCommand, + subscribeSpeechGenerationCommands, +} from '../../api/speechGenerationCommands' +import i18n, { useUiTranslation } from '../../i18n' +import { + SPEECH_PRESENTATION_EVENT, + SPEECH_RESULT_EVENT, + type SpeechPresentation, +} from './speechCommandPresentation' +import type { StudioSpeechGenerationCommand } from './speechGenerationSpec' + +function parameters(command: StudioSpeechGenerationCommand): Record { + return command.input.params as Record +} + +function referenceCount(params: Record): number { + return [ + 'audio_guide', + 'audio_guide2', + 'audio_guide3', + 'audio_guide4', + 'audio_guide5', + 'audio_guide6', + ].reduce((count, field) => count + (params[field] !== undefined && params[field] !== null && params[field] !== '' ? 1 : 0), 0) +} + +function RequestSummary({ command }: { command: StudioSpeechGenerationCommand }) { + const { t } = useUiTranslation('studio') + const params = parameters(command) + const originalPrompt = typeof params._tts_original_prompt === 'string' + ? params._tts_original_prompt + : params.prompt + // Zero is a native auto-duration sentinel for models such as DramaBox; + // showing "0s" would make a valid restored request look empty. Keep null + // and omitted values on the same display path until model preflight fills + // its effective default. + const duration = typeof params.duration_seconds === 'number' && params.duration_seconds > 0 + ? `${params.duration_seconds}s` + : t('speechCommands.autoDuration') + const voices = typeof params._tts_voice_count === 'number' ? params._tts_voice_count : 0 + const loras = Array.isArray(params.activated_loras) ? params.activated_loras.length : 0 + return
+
{String(params.model_type)} · {duration} · {command.input.workspace}
+

{String(originalPrompt ?? '')}

+
{t('speechCommands.resources', { + references: referenceCount(params), voices, loras, + })}
+
+} + +interface Props { + workspace: string + model: string + visible: boolean + onRecovered: (receipt: SpeechGenerationReceipt) => Promise +} + +export function StudioSpeechCommandPanel({ workspace, model, visible, onRecovered }: Props) { + const { t } = useUiTranslation('studio') + const [shown, setShown] = useState(null) + const [pending, setPending] = useState([]) + const [error, setError] = useState('') + const [busy, setBusy] = useState(false) + const [receipt, setReceipt] = useState(null) + const current = useRef({ workspace, model, visible }) + const root = useRef(null) + const waiting = useRef(null) + useLayoutEffect(() => { current.current = { workspace, model, visible } }, [workspace, model, visible]) + + useEffect(() => { + const refresh = () => { + try { setPending(pendingSpeechGenerationCommands(workspace)) } + catch { setError(i18n.t('studio:speechCommands.pendingInvalid')) } + } + refresh() + return subscribeSpeechGenerationCommands(refresh) + }, [workspace]) + + useLayoutEffect(() => { + const elementAtSetup = root.current + const receive = (event: Event) => { + const request = (event as CustomEvent).detail + const view = current.current + if (!view.visible || request.command.input.workspace !== view.workspace + || String(parameters(request.command).model_type) !== view.model || waiting.current?.active) { + request.respond(i18n.t('studio:speechCommands.contextChanged')) + return + } + waiting.current = request + setShown(request.command) + setBusy(true) + setReceipt(null) + setError('') + } + window.addEventListener(SPEECH_PRESENTATION_EVENT, receive) + if (elementAtSetup) elementAtSetup.dataset.studioSpeechListening = 'true' + // React StrictMode can simulate an effect cleanup while retaining the DOM + // node. Defer rejection until after that probe; real navigation removes + // the node and still receives a deterministic pre-POST failure. + return () => { + if (elementAtSetup) elementAtSetup.dataset.studioSpeechListening = 'false' + const request = waiting.current + queueMicrotask(() => { + if (request?.active && (!elementAtSetup || !elementAtSetup.isConnected)) { + request.respond(i18n.t('studio:speechCommands.panelUnavailable')) + } + }) + window.removeEventListener(SPEECH_PRESENTATION_EVENT, receive) + } + }, []) + + useEffect(() => { + const complete = (event: Event) => { + const result = (event as CustomEvent<{ intentId: string; receipt?: SpeechGenerationReceipt; error?: string }>).detail + if (result.intentId !== shown?.intent_id) return + setBusy(false) + setReceipt(result.receipt || null) + setError(result.error || '') + } + window.addEventListener(SPEECH_RESULT_EVENT, complete) + return () => window.removeEventListener(SPEECH_RESULT_EVENT, complete) + }, [shown]) + + useLayoutEffect(() => { + const request = waiting.current + if (!request?.active || request.command !== shown || !root.current) return + root.current.dataset.studioSpeechCommand = request.command.intent_id + root.current.scrollIntoView?.({ block: 'nearest' }) + let second = 0 + const first = requestAnimationFrame(() => { + second = requestAnimationFrame(() => { + const view = current.current + const valid = root.current?.isConnected && view.visible + && view.workspace === request.command.input.workspace + && view.model === String(parameters(request.command).model_type) + request.respond(valid ? undefined : i18n.t('studio:speechCommands.contextChanged')) + waiting.current = null + }) + }) + return () => { cancelAnimationFrame(first); cancelAnimationFrame(second) } + }, [shown]) + + const recover = async (command: StudioSpeechGenerationCommand) => { + setBusy(true) + setError('') + setShown(command) + try { + const admitted = await submitSpeechGenerationCommand(command) + setReceipt(admitted) + await onRecovered(admitted) + } catch (failure) { + setError(failure instanceof Error ? failure.message : String(failure)) + } finally { setBusy(false) } + } + + return
+ {shown &&
+ {receipt ? t('speechCommands.admitted', { id: receipt.result.job_id }) : t('speechCommands.prepared')} + +
} + {pending.filter(command => !busy || command.intent_id !== shown?.intent_id).map(command =>
+ {t('speechCommands.pending')} + + +
)} + {error &&

{error}

} +
+} diff --git a/ui/src/features/studio/actions.ts b/ui/src/features/studio/actions.ts index bac8f9d15..bec8a61cb 100644 --- a/ui/src/features/studio/actions.ts +++ b/ui/src/features/studio/actions.ts @@ -1,5 +1,5 @@ import * as api from '../../api/client' -import type { ImageGenerationReceipt } from '../../api/imageGenerationCommands' +import type { GenerationReceiptLike } from '../../api/generationCommandClient' import i18n from '../../i18n' import { commandResultFromSlice, type CommandResult } from '../../lib/commandContract' import { getFamiliesForMode, getModelsForFamily, useStore } from '../../stores/useStore' @@ -355,7 +355,7 @@ export async function prepareAudio(action: PrepareAudioCommand): Promise void +} + +export function finishStudioSpeechCommand( + intentId: string, + receipt?: SpeechGenerationReceipt, + error?: string, +): void { + window.dispatchEvent(new CustomEvent(SPEECH_RESULT_EVENT, { detail: { intentId, receipt, error } })) +} + +async function mountedSpeechPanel(): Promise { + window.dispatchEvent(new Event('hocuspocus:studio-speech-open')) + const find = () => document.querySelector( + '[data-studio-speech-ready="true"][data-studio-speech-listening="true"]', + ) + const current = find() + if (current) return current + return new Promise((resolve, reject) => { + const timer = window.setTimeout(() => { + observer.disconnect() + reject(new Error(i18n.t('studio:commands.panelUnavailable'))) + }, 8000) + const observer = new MutationObserver(() => { + const panel = find() + if (panel) { + observer.disconnect() + clearTimeout(timer) + resolve(panel) + } + }) + observer.observe(document.body, { + childList: true, + subtree: true, + attributes: true, + attributeFilter: ['data-studio-speech-ready', 'data-studio-speech-listening'], + }) + }) +} + +/** Wait for a visible React commit of this exact Speech request before POST. */ +export async function presentStudioSpeechCommand(command: StudioSpeechGenerationCommand): Promise { + const root = await mountedSpeechPanel() + await new Promise((resolve, reject) => { + const observer = new MutationObserver(() => { + if (!root.isConnected) request.respond(i18n.t('studio:commands.panelUnavailable')) + }) + const request: SpeechPresentation = { + command: structuredClone(command), + active: true, + respond: error => { + if (!request.active) return + request.active = false + clearTimeout(timer) + observer?.disconnect() + if (error) reject(new Error(error)) + else resolve() + }, + } + const timer = window.setTimeout(() => request.respond(i18n.t('studio:commands.panelUnavailable')), 8000) + observer.observe(document.body, { childList: true, subtree: true }) + window.dispatchEvent(new CustomEvent(SPEECH_PRESENTATION_EVENT, { detail: request })) + }) + if (!root.isConnected || root.dataset.studioSpeechCommand !== command.intent_id) { + throw new Error(i18n.t('studio:commands.contextChanged')) + } +} diff --git a/ui/src/features/studio/speechCommandSubmission.ts b/ui/src/features/studio/speechCommandSubmission.ts new file mode 100644 index 000000000..5b62a626b --- /dev/null +++ b/ui/src/features/studio/speechCommandSubmission.ts @@ -0,0 +1,178 @@ +import * as api from '../../api/client' +import { BASE } from '../../api/http' +import { stableSerialize } from '../../lib/commandContract' +import type { AppState } from '../../stores/useStore' +import type { GenerationSubmissionContext } from './generationProvenance' +import { + createStudioSpeechGenerationCommand, + projectStudioSpeechFormParams, + type StudioSpeechGenerationCommand, +} from './speechGenerationSpec' +import { + finishStudioSpeechCommand, + presentStudioSpeechCommand, +} from './speechCommandPresentation' +import { + newSpeechGenerationIntentId, + submitSpeechGenerationCommand, + type SpeechGenerationReceipt, +} from '../../api/speechGenerationCommands' +import i18n from '../../i18n' + +type StudioState = AppState +type NativeReceipt = Awaited> + +export interface SpeechSubmission { + params: Record + receipt?: SpeechGenerationReceipt + submit: () => Promise +} + +const AUDIO_FIELDS = [ + 'audio_guide', + 'audio_guide2', + 'audio_guide3', + 'audio_guide4', + 'audio_guide5', + 'audio_guide6', +] as const + +type VoiceSnapshot = { + name: string + filename: string | null + path: string | null +} + +function voiceSnapshot(state: StudioState): VoiceSnapshot[] { + return state.ttsVoices.map(voice => ({ + name: voice.name, + filename: voice.filename, + path: voice.path, + })) +} + +/** + * Speech has controls outside `params`. Include each one in the admission + * guard so editing a voice, its file or duration while refs are resolving can + * never post a snapshot that no longer describes the visible form. + */ +function speechFormFingerprint(state: StudioState): string { + return stableSerialize({ + params: state.params, + activeWorkspace: state.activeWorkspace, + generationMode: state.generationMode, + audioSubMode: state.audioSubMode, + durationSeconds: state.durationSeconds, + ttsVoiceCount: state.ttsVoiceCount, + ttsSpeakerName1: state.ttsSpeakerName1, + ttsSpeakerName2: state.ttsSpeakerName2, + ttsVoices: voiceSnapshot(state), + settingsOpen: state.settingsOpen, + dashboardOpen: state.dashboardOpen, + sidebarMode: state.sidebarMode, + }) +} + +function assertSameSpeechForm(before: StudioState, current: StudioState): void { + if (speechFormFingerprint(before) !== speechFormFingerprint(current)) { + throw new Error(i18n.t('studio:commands.contextChanged')) + } +} + +/** Convert legacy upload paths to canonical references without reading files. */ +async function canonicalAudioReferences(params: Record): Promise { + const fields = AUDIO_FIELDS.filter(field => { + const value = params[field] + return value !== undefined && value !== null && value !== '' + }) + // Do this validation before constructing the compact reference request. A + // restored form can contain a malformed value in one voice slot while a + // later slot is valid. Filtering non-strings would shift the later value + // into the earlier slot and silently attach the wrong voice after resolve. + for (const field of fields) { + if (typeof params[field] !== 'string') { + throw new Error(`${field} must be a string audio reference`) + } + } + const references = fields.map(field => params[field] as string) + if (!references.length) return + const response = await fetch(`${BASE}/api/v1/generation/commands/references`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ references, media_kind: 'audio' }), + }) + if (!response.ok) { + const body = await response.json().catch(() => ({})) as { detail?: { message?: string } } + throw new Error(body.detail?.message || i18n.t('studio:commands.referenceFailed')) + } + const result = await response.json() as { references?: unknown } + if (!Array.isArray(result.references) || result.references.length !== references.length + || result.references.some(value => typeof value !== 'string')) { + throw new Error(i18n.t('studio:commands.referenceFailed')) + } + const resolved = result.references as string[] + fields.forEach((field, index) => { params[field] = resolved[index] }) +} + +function nativeParams(command: StudioSpeechGenerationCommand): Record { + return { ...command.input.params, workspace: command.input.workspace } +} + +/** Route Speech through the durable generation.speech ACK/receipt gateway. */ +export async function prepareStudioSpeechSubmission( + params: Record, + before: StudioState, + current: () => StudioState, + context?: GenerationSubmissionContext, + referenceErrors: string[] = [], +): Promise { + if (before.generationMode !== 'audio' || before.audioSubMode !== 'speech') { + return { params, submit: () => api.submitGeneration(params) } + } + + let snapshotParams = params + try { + if (referenceErrors.length) throw new Error(i18n.t('studio:commands.referenceFailed')) + snapshotParams = JSON.parse(stableSerialize(params)) as Record + // Load Settings/reroll restores the shared Studio form, which can carry + // known video/H3 controls alongside the speech fields. Project only that + // explicit form residue; the command builder remains closed for direct + // Wizard/MCP envelopes and rejects every other unknown key. + snapshotParams = projectStudioSpeechFormParams(snapshotParams).params + assertSameSpeechForm(before, current()) + await canonicalAudioReferences(snapshotParams) + assertSameSpeechForm(before, current()) + const command = createStudioSpeechGenerationCommand( + snapshotParams, + context?.commandId || newSpeechGenerationIntentId(), + ) + const submission: SpeechSubmission = { + params: nativeParams(command), + submit: async () => { + try { + const receipt = await submitSpeechGenerationCommand(command, { + submissionContext: context, + onSnapshotReady: async frozen => { + assertSameSpeechForm(before, current()) + await presentStudioSpeechCommand(frozen) + assertSameSpeechForm(before, current()) + }, + }) + submission.receipt = receipt + finishStudioSpeechCommand(command.intent_id, receipt) + return { ...receipt.result, status: receipt.status } + } catch (error) { + finishStudioSpeechCommand( + command.intent_id, + undefined, + error instanceof Error ? error.message : String(error), + ) + throw error + } + }, + } + return submission + } catch (error) { + return { params: snapshotParams, submit: () => Promise.reject(error) } + } +} diff --git a/ui/src/features/studio/speechGenerationSpec.ts b/ui/src/features/studio/speechGenerationSpec.ts new file mode 100644 index 000000000..c1e8534ba --- /dev/null +++ b/ui/src/features/studio/speechGenerationSpec.ts @@ -0,0 +1,530 @@ +import { stableSerialize } from '../../lib/commandContract' +import speechCommandCatalog from '../../api/speechCommandCatalog.json' + +export const STUDIO_SPEECH_SCHEMA_VERSION = 2 as const +export const STUDIO_SPEECH_OPERATION = 'generation.speech' as const + +type CatalogRecord = Record +type SpeechSchema = CatalogRecord & { + anyOf?: unknown[] + properties?: CatalogRecord + required?: string[] + additionalProperties?: boolean + $defs?: CatalogRecord + $ref?: string + const?: unknown + enum?: unknown[] + type?: string + minLength?: number + maxLength?: number + pattern?: string + minimum?: number + maximum?: number + minItems?: number + maxItems?: number + items?: unknown +} + +const studioCatalog = speechCommandCatalog.studio as unknown as CatalogRecord +const inputSchema = studioCatalog.input as SpeechSchema +const paramsSchema = (inputSchema.$defs?.StudioSpeechParams || {}) as SpeechSchema + +/** + * The generated catalog is the sole source of the speech parameter allowlist. + * Keeping the key type derived from its JSON object also makes adding a native + * field a catalog change instead of a second hand-maintained client schema. + */ +export type StudioSpeechParamKey = keyof typeof speechCommandCatalog.studio.input.$defs.StudioSpeechParams.properties +export type StudioSpeechParams = Partial> +export type StudioSpeechGenerationFullParams = StudioSpeechParams & { + workspace: string + provenance?: unknown +} + +export interface StudioSpeechGenerationInput { + workspace: string + workspace_collection_id?: string | null + params: StudioSpeechParams +} + +export interface StudioSpeechGenerationCommand { + version: typeof STUDIO_SPEECH_SCHEMA_VERSION + operation: typeof STUDIO_SPEECH_OPERATION + intent_id: string + input: StudioSpeechGenerationInput +} + +export const STUDIO_SPEECH_PARAM_KEYS: readonly StudioSpeechParamKey[] = + studioCatalog.supported_input_fields as StudioSpeechParamKey[] +export const STUDIO_SPEECH_PARAM_CATALOG: ReadonlySet = new Set( + STUDIO_SPEECH_PARAM_KEYS as readonly string[], +) + +/** + * Fields which can remain in the shared Studio form after Load Settings or a + * reroll has restored a video/H3 sidecar. They are deliberately listed + * rather than matched by a prefix: a form projection may remove these known + * UI-only leftovers, while the command builder below must continue to reject + * an unknown key supplied directly by a Wizard/MCP caller. + * + * Values for fields in the speech catalog are never removed here, including + * explicit inactive sentinels. The native speech schema remains responsible + * for rejecting an active image/video/H3 value that is invalid for speech. + */ +export const STUDIO_SPEECH_FORM_RESIDUAL_FIELDS = [ + // Shared video/image controls restored into the common Studio params map. + 'viggle_audio_mode', + 'switch_threshold', + 'denoising_strength', + 'video_guide_outpainting', + 'sliding_window_size', + 'sliding_window_overlap', + 'sliding_window_memory_override', + 'sliding_window_discard_last_frames', + 'sliding_window_color_correction_strength', + 'sliding_window_overlap_noise', + 'keep_frames_video_source', + 'keep_frames_video_guide', + 'force_fps', + 'skip_steps_cache_type', + 'skip_steps_multiplier', + 'skip_steps_start_step_perc', + 'video_prompt_type', + 'image_prompt_type', + 'input_video_strength', + 'preserve_source_style', + 'image_fit_mode', + 'frames_positions', + 'injection_strength', + 'remove_background_images_ref', + 'self_refiner_setting', + 'per_clip_frames', + 'per_clip_keyframes', + // H3 video policy and window controls. Reference lists themselves are + // catalogued inactive fields and therefore remain subject to schema checks. + 'h3_audio_shift', + 'h3_audio_prompt', + 'h3_ref_image_size', + 'h3_reference_mode', + 'h3_model_profile', + 'minimax_h3_reference_detail', + 'minimax_h3_text_encoder', + 'minimax_h3_turbo_preset', + 'minimax_h3_planning_style', + 'minimax_h3_audio_policy', + 'minimax_h3_reference_sequence', + 'minimax_h3_semantic_bridge_alpha', + 'minimax_h3_semantic_bridge_magnitude', + 'minimax_h3_multi_window', + 'h3_reference_context', + 'minimax_h3_window_storyboard', + 'h3_window_prompts', + 'h3_window_plan_signature', + 'h3_window_plan', + // Restored advanced pipeline toggles and model-specific controls. + 'stage2_steps', + 'progressive_pipeline', + 'single_stage_pipeline', + 'reference_pipeline', + 'progressive_stage1_image_weight', + 'progressive_stage2_steps', + 'progressive_stage2_sigma', + 'progressive_stage3_steps', + 'progressive_stage3_sigma', + 'progressive_stage3_image_weight', + 'stg_scale', + 'perturbation_switch', + 'perturbation_layers', + 'perturbation_start_perc', + 'perturbation_end_perc', + 'cfg_rescale', + 'modality_scale', + 'use_gradient_estimation', + 'ge_gamma', + 'ge_alpha', + 'keyframe_conditioning_mode', + 'keyframe_inject_mode', + 'override_attention', + 'attention_sparsity', + // Other known form/sidecar controls from non-speech modes. + 'batch_size', + 'tts_voice_count', + 'voice_clone_enabled', + 'voice_clone_mode', + 'voice_clone_refs', + 'voice_reference', + 'identity_guidance_scale', + 'speakers_locations', + 'video_guide2', + 'sfx_mode', + '_sfx_virtual_model', + '_mmaudio_variant', + '_music_description', + '_music_instrumental', + 'edit_sub_mode', +] as const + +const STUDIO_SPEECH_FORM_RESIDUAL_SET: ReadonlySet = new Set( + STUDIO_SPEECH_FORM_RESIDUAL_FIELDS, +) + +export interface StudioSpeechFormProjection { + params: Record + droppedFields: string[] +} + +/** + * Project the complete in-memory Studio form into the Speech command input. + * + * This is intentionally a caller-side adapter, not a relaxation of the + * closed command contract. Direct builders still reject every unrecognised + * key, including a key which merely resembles one of the residual fields. + */ +export function projectStudioSpeechFormParams( + fullParams: Record, +): StudioSpeechFormProjection { + if (!isRecord(fullParams)) throw new Error('Studio speech parameters must be an object') + const params: Record = {} + const droppedFields: string[] = [] + for (const [key, value] of Object.entries(fullParams)) { + if (value === undefined) continue + if (key === 'workspace' || DECLARED_METADATA_FIELDS.has(key)) { + params[key] = value + continue + } + if (ENVELOPE_INJECTION_FIELDS.has(key)) { + throw new Error('workspace parameters cannot contain envelope field ' + key) + } + if (STUDIO_SPEECH_PARAM_CATALOG.has(key)) { + params[key] = value + continue + } + if (STUDIO_SPEECH_FORM_RESIDUAL_SET.has(key)) { + droppedFields.push(key) + continue + } + throw new Error('input.params.' + key + ' is not supported by generation.speech') + } + return { params, droppedFields } +} + +const COMMAND_FIELDS = new Set(['version', 'operation', 'intent_id', 'input']) +const INPUT_FIELDS = new Set(['workspace', 'workspace_collection_id', 'params']) +const DECLARED_METADATA_FIELDS = new Set([ + 'provenance', + 'runtime', + 'client', + 'actor', + 'permission', + 'workspace_id', + 'workspaceId', +]) +const ENVELOPE_INJECTION_FIELDS = new Set([ + 'version', + 'operation', + 'intent_id', + 'input', + 'params', + 'command', + 'command_id', + 'commandId', +]) +const AUDIO_REFERENCE_FIELDS = [ + 'audio_guide', + 'audio_guide2', + 'audio_guide3', + 'audio_guide4', + 'audio_guide5', + 'audio_guide6', +] as const +const ASSET_ID = /^asset(?:[_:-])[A-Za-z0-9][A-Za-z0-9._:-]{0,238}$/ +const WORKSPACE = /^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$/ +const MAX_INTENT_LENGTH = 160 +const MAX_WORKSPACE_LENGTH = 240 +const MAX_COLLECTION_LENGTH = 200 + +function isRecord(value: unknown): value is CatalogRecord { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false + const prototype = Object.getPrototypeOf(value) + return prototype === Object.prototype || prototype === null +} + +function requiredText(value: unknown, field: string, maximum: number): string { + if (typeof value !== 'string' || !value.trim()) throw new Error(`${field} must be a non-blank string`) + if (value.length > maximum) throw new Error(`${field} is too long`) + return value +} + +function safeReferencePath(value: string): boolean { + return Boolean(value) + && !value.includes('\\') + && !value.includes('\u0000') + && value.split('/').every(part => Boolean(part) && part !== '.' && part !== '..') +} + +/** Match the server's canonical asset URL/ID syntax without resolving anything. */ +export function assertCanonicalAudioReference(value: unknown, field: string): void { + if (value === null || value === '') return + if (typeof value !== 'string' || value.trim() !== value) { + throw new Error(`${field} must be a canonical audio URL or asset ID`) + } + if (ASSET_ID.test(value)) return + if (!value.startsWith('/api/v1/') || value.includes('\\') || value.includes('\u0000')) { + throw new Error(`${field} must be a canonical audio URL or asset ID`) + } + const rawPathEnd = value.search(/[?#]/) + const rawPath = rawPathEnd < 0 ? value : value.slice(0, rawPathEnd) + let decodedPath: string + try { + decodedPath = decodeURIComponent(rawPath) + } catch { + throw new Error(`${field} must be a canonical audio URL or asset ID`) + } + const parsed = new URL(value, 'http://hocuspocus.invalid') + if (parsed.origin !== 'http://hocuspocus.invalid' || parsed.hash) { + throw new Error(`${field} must be a canonical audio URL or asset ID`) + } + if (parsed.pathname.startsWith('/api/v1/uploads/')) { + if (parsed.search || !safeReferencePath(decodedPath.slice('/api/v1/uploads/'.length))) { + throw new Error(`${field} must be a canonical audio URL or asset ID`) + } + return + } + if (parsed.pathname.startsWith('/api/v1/assets/')) { + const suffix = decodedPath.slice('/api/v1/assets/'.length) + if (parsed.search || !ASSET_ID.test(suffix)) { + throw new Error(`${field} must be a canonical audio URL or asset ID`) + } + return + } + if (parsed.pathname.startsWith('/api/v1/file/')) { + const suffix = decodedPath.slice('/api/v1/file/'.length) + const query = new URLSearchParams(parsed.search) + if (!safeReferencePath(suffix) || query.size !== 1 || query.getAll('workspace').length !== 1 + || !WORKSPACE.test(query.get('workspace') || '')) { + throw new Error(`${field} must be a canonical audio URL or asset ID`) + } + return + } + throw new Error(`${field} must be a canonical audio URL or asset ID`) +} + +function schemaTypeMatches(value: unknown, type: unknown): boolean { + if (type === 'null') return value === null + if (type === 'string') return typeof value === 'string' + if (type === 'boolean') return typeof value === 'boolean' + if (type === 'integer') return typeof value === 'number' && Number.isSafeInteger(value) + if (type === 'number') return typeof value === 'number' && Number.isFinite(value) + if (type === 'array') return Array.isArray(value) + if (type === 'object') return isRecord(value) + return true +} + +function resolveSchema(schema: unknown): SpeechSchema { + if (!isRecord(schema)) throw new Error('The generated speech catalog contains an invalid schema') + if (typeof schema.$ref !== 'string') return schema + const prefix = '#/$defs/' + if (!schema.$ref.startsWith(prefix)) throw new Error('The generated speech catalog contains an unsupported reference') + return resolveSchema(paramsSchema.$defs?.[schema.$ref.slice(prefix.length)]) +} + +function assertCatalogUnion(value: unknown, schema: SpeechSchema, field: string): void { + const valid = schema.anyOf?.some(option => { + try { + assertCatalogValue(value, option, field) + return true + } catch { + return false + } + }) + if (!valid) throw new Error(`${field} has an invalid native speech value`) +} + +function assertCatalogIdentity(value: unknown, schema: SpeechSchema, field: string): void { + if ('const' in schema && value !== schema.const) { + throw new Error(`${field} must equal its native selector`) + } + if (Array.isArray(schema.enum) && !schema.enum.some(option => Object.is(option, value))) { + throw new Error(`${field} has an invalid native speech value`) + } + if (schema.type && !schemaTypeMatches(value, schema.type)) { + throw new Error(`${field} has an invalid native speech type`) + } +} + +function assertCatalogString(value: unknown, schema: SpeechSchema, field: string): void { + if (typeof value !== 'string') return + if (typeof schema.minLength === 'number' && value.length < schema.minLength) { + throw new Error(`${field} is too short`) + } + if (typeof schema.maxLength === 'number' && value.length > schema.maxLength) { + throw new Error(`${field} is too long`) + } + if (typeof schema.pattern === 'string' && !(new RegExp(schema.pattern).test(value))) { + throw new Error(`${field} has an invalid native speech value`) + } +} + +function assertCatalogNumber(value: unknown, schema: SpeechSchema, field: string): void { + if (typeof value !== 'number') return + if (!Number.isFinite(value)) throw new Error(`${field} must be finite`) + if (typeof schema.minimum === 'number' && value < schema.minimum) { + throw new Error(`${field} is below its native minimum`) + } + if (typeof schema.maximum === 'number' && value > schema.maximum) { + throw new Error(`${field} is above its native maximum`) + } +} + +function assertCatalogArray(value: unknown, schema: SpeechSchema, field: string): void { + if (!Array.isArray(value)) return + if (typeof schema.minItems === 'number' && value.length < schema.minItems) { + throw new Error(`${field} has too few items`) + } + if (typeof schema.maxItems === 'number' && value.length > schema.maxItems) { + throw new Error(`${field} has too many items`) + } + if (schema.items) { + value.forEach((item, index) => assertCatalogValue(item, schema.items, `${field}[${index}]`)) + } +} + +function assertCatalogObject(value: unknown, schema: SpeechSchema, field: string): void { + if (!isRecord(value)) return + const properties = schema.properties || {} + if (schema.additionalProperties === false) { + for (const key of Object.keys(value)) { + if (!(key in properties)) throw new Error(`${field}.${key} is not supported by generation.speech`) + } + } + if (Array.isArray(schema.required)) { + for (const key of schema.required) { + if (!(key in value)) throw new Error(`${field}.${key} is required by generation.speech`) + } + } + for (const [key, child] of Object.entries(value)) { + const childSchema = properties[key] + if (childSchema) assertCatalogValue(child, childSchema, `${field}.${key}`) + } +} + +function assertCatalogValue(value: unknown, schema: unknown, field: string): void { + const resolved = resolveSchema(schema) + if (Array.isArray(resolved.anyOf)) { + assertCatalogUnion(value, resolved, field) + return + } + assertCatalogIdentity(value, resolved, field) + assertCatalogString(value, resolved, field) + assertCatalogNumber(value, resolved, field) + assertCatalogArray(value, resolved, field) + assertCatalogObject(value, resolved, field) +} + +function assertSpeechParams(value: unknown): asserts value is StudioSpeechParams { + if (!isRecord(value)) throw new Error('input.params must be an object') + for (const key of Object.keys(value)) { + if (!STUDIO_SPEECH_PARAM_CATALOG.has(key)) { + throw new Error(`input.params.${key} is not supported by generation.speech`) + } + } + assertCatalogValue(value, paramsSchema, 'input.params') + if (value.generation_mode !== undefined && value.generation_mode !== 'audio') { + throw new Error('input.params.generation_mode must be audio') + } + if (value._audio_sub_mode !== undefined && value._audio_sub_mode !== 'speech') { + throw new Error('input.params._audio_sub_mode must be speech') + } + if (value._tts_original_prompt === null) { + throw new Error('input.params._tts_original_prompt must be a string when supplied') + } + if (value.minimax_h3_turbo_mode === true) { + throw new Error('input.params.minimax_h3_turbo_mode must be false or null in speech mode') + } + for (const field of AUDIO_REFERENCE_FIELDS) { + if (field in value) assertCanonicalAudioReference(value[field], `input.params.${field}`) + } + if (Array.isArray(value.activated_loras)) { + value.activated_loras.forEach((item, index) => { + if (!item.trim() || item.includes('/') || item.includes('\\')) { + throw new Error(`input.params.activated_loras[${index}] must be an exact catalog name`) + } + }) + } + // stableSerialize is the JSON boundary: it rejects cycles, BigInt, class + // instances, functions and non-finite values before localStorage or fetch. + stableSerialize(value) +} + +export function assertStudioSpeechGenerationCommand( + value: unknown, +): asserts value is StudioSpeechGenerationCommand { + if (!isRecord(value)) throw new Error('Studio speech generation command must be an object') + for (const key of Object.keys(value)) { + if (!COMMAND_FIELDS.has(key)) throw new Error(`command.${key} is not supported by generation.speech`) + } + if (value.version !== STUDIO_SPEECH_SCHEMA_VERSION) throw new Error('version must be the integer 2') + if (value.operation !== STUDIO_SPEECH_OPERATION) throw new Error('operation must be generation.speech') + requiredText(value.intent_id, 'intent_id', MAX_INTENT_LENGTH) + if (!isRecord(value.input)) throw new Error('input must be an object') + for (const key of Object.keys(value.input)) { + if (!INPUT_FIELDS.has(key)) throw new Error(`input.${key} is not supported by generation.speech`) + } + const workspace = requiredText(value.input.workspace, 'input.workspace', MAX_WORKSPACE_LENGTH) + if (!WORKSPACE.test(workspace)) throw new Error('input.workspace must be an exact output workspace name') + if ('workspace_collection_id' in value.input && value.input.workspace_collection_id !== null) { + requiredText(value.input.workspace_collection_id, 'input.workspace_collection_id', MAX_COLLECTION_LENGTH) + } + assertSpeechParams(value.input.params) +} + +export function detachedStudioSpeechGenerationCommand(value: unknown): StudioSpeechGenerationCommand { + assertStudioSpeechGenerationCommand(value) + return JSON.parse(stableSerialize(value)) as StudioSpeechGenerationCommand +} + +function takeWorkspace(value: CatalogRecord): string { + const workspace = requiredText(value.workspace, 'workspace', MAX_WORKSPACE_LENGTH) + if (!WORKSPACE.test(workspace)) throw new Error('workspace must be an exact output workspace name') + return workspace +} + +function takeWorkspaceCollectionId(value: unknown): string | undefined { + if (value === undefined || value === null) return undefined + if (!isRecord(value)) throw new Error('provenance must be an object when supplied') + if (value.workspace_id === undefined || value.workspace_id === null) return undefined + return requiredText(value.workspace_id, 'provenance.workspace_id', MAX_COLLECTION_LENGTH) +} + +/** Build a detached Speech envelope from the complete native Studio form. */ +export function createStudioSpeechGenerationCommand( + fullParams: Record, + intentId: string, +): StudioSpeechGenerationCommand { + if (!isRecord(fullParams)) throw new Error('Studio speech parameters must be an object') + const workspace = takeWorkspace(fullParams) + const workspaceCollectionId = takeWorkspaceCollectionId(fullParams.provenance) + const params: Record = {} + for (const [key, value] of Object.entries(fullParams)) { + if (key === 'workspace' || DECLARED_METADATA_FIELDS.has(key)) continue + if (ENVELOPE_INJECTION_FIELDS.has(key)) { + throw new Error('workspace parameters cannot contain envelope field ' + key) + } + if (!STUDIO_SPEECH_PARAM_CATALOG.has(key)) { + throw new Error('input.params.' + key + ' is not supported by generation.speech') + } + params[key] = value + } + return detachedStudioSpeechGenerationCommand({ + version: STUDIO_SPEECH_SCHEMA_VERSION, + operation: STUDIO_SPEECH_OPERATION, + intent_id: intentId, + input: { + workspace, + ...(workspaceCollectionId !== undefined ? { workspace_collection_id: workspaceCollectionId } : {}), + params: params as StudioSpeechParams, + }, + }) +} + +export const buildStudioSpeechGenerationCommand = createStudioSpeechGenerationCommand diff --git a/ui/src/features/studio/studioSubmission.ts b/ui/src/features/studio/studioSubmission.ts index 489a6a95d..334112a8e 100644 --- a/ui/src/features/studio/studioSubmission.ts +++ b/ui/src/features/studio/studioSubmission.ts @@ -3,20 +3,35 @@ import * as api from '../../api/client' type Preparation = typeof import('./imageCommandSubmission').prepareStudioSubmission type Inputs = Parameters type Loader = () => Promise<{ prepareStudioSubmission: Preparation }> +type SpeechPreparation = typeof import('./speechCommandSubmission').prepareStudioSpeechSubmission +type StudioSubmission = Awaited> | Awaited> /** Preserve the legacy path and surface image chunk failures in the job tile. */ export async function prepareStudioSubmission( params: Inputs[0], before: Inputs[1], current: Inputs[2], context?: Inputs[3], referenceErrors?: Inputs[4], load: Loader = () => import('./imageCommandSubmission'), -): ReturnType { - if (before.generationMode !== 'image') return { params, submit: () => api.submitGeneration(params) } - try { - const implementation = await load() - return await implementation.prepareStudioSubmission(params, before, current, context, referenceErrors) - } catch (error) { - return { params, submit: () => Promise.reject(error) } +): Promise { + if (before.generationMode === 'image') { + try { + const implementation = await load() + return await implementation.prepareStudioSubmission(params, before, current, context, referenceErrors) + } catch (error) { + return { params, submit: () => Promise.reject(error) } + } } + // Speech gets the same durable snapshot/ACK boundary as typed image jobs. + // Keep this import inside the selected sub-mode so opening Studio Image or + // another audio tool does not pull the speech contract into the main chunk. + if (before.generationMode === 'audio' && before.audioSubMode === 'speech') { + try { + const implementation = await import('./speechCommandSubmission') + return await implementation.prepareStudioSpeechSubmission(params, before, current, context, referenceErrors) + } catch (error) { + return { params, submit: () => Promise.reject(error) } + } + } + return { params, submit: () => api.submitGeneration(params) } } /** Image references travel as canonical URLs; other native modes retain paths. */ diff --git a/ui/src/i18n/locales/en/studio.json b/ui/src/i18n/locales/en/studio.json index 2dee93e4d..0c69239db 100644 --- a/ui/src/i18n/locales/en/studio.json +++ b/ui/src/i18n/locales/en/studio.json @@ -938,5 +938,17 @@ "pendingInvalid": "The saved submission could not be read. It has been preserved for recovery.", "referenceFailed": "A selected reference could not be prepared. No image was submitted.", "conflictingGuides": "Conflicting image guides are selected. Choose one control image and mask before generating." + }, + "speechCommands": { + "prepared": "Speech request ready", + "admitted": "Speech queued · {{id}}", + "pending": "A previous speech submission needs recovery", + "recover": "Recover this speech submission", + "resources": "{{references}} audio references · {{voices}} voices · {{loras}} LoRAs", + "autoDuration": "auto duration", + "contextChanged": "The speech form changed before submission. Review the current settings and try again.", + "panelUnavailable": "The speech request could not be shown. Open Studio and try again.", + "pendingInvalid": "The saved speech submission could not be read. It has been preserved for recovery.", + "referenceFailed": "A selected audio reference could not be prepared. No speech was submitted." } } diff --git a/ui/src/i18n/locales/es/studio.json b/ui/src/i18n/locales/es/studio.json index c6b128802..a8601110a 100644 --- a/ui/src/i18n/locales/es/studio.json +++ b/ui/src/i18n/locales/es/studio.json @@ -938,5 +938,17 @@ "pendingInvalid": "No se pudo leer la solicitud guardada. Se ha conservado para recuperarla.", "referenceFailed": "No se pudo preparar una referencia seleccionada. No se ha enviado ninguna imagen.", "conflictingGuides": "Hay guías de imagen distintas seleccionadas. Elige una imagen de control y una máscara antes de generar." + }, + "speechCommands": { + "prepared": "Solicitud de voz preparada", + "admitted": "Voz en cola · {{id}}", + "pending": "Una solicitud de voz anterior necesita recuperación", + "recover": "Recuperar esta solicitud de voz", + "resources": "{{references}} referencias de audio · {{voices}} voces · {{loras}} LoRAs", + "autoDuration": "duración automática", + "contextChanged": "El formulario de voz cambió antes del envío. Revisa los ajustes actuales y vuelve a intentarlo.", + "panelUnavailable": "No se pudo mostrar la solicitud de voz. Abre Studio y vuelve a intentarlo.", + "pendingInvalid": "No se pudo leer la solicitud de voz guardada. Se ha conservado para recuperarla.", + "referenceFailed": "No se pudo preparar una referencia de audio seleccionada. No se ha enviado ninguna voz." } } diff --git a/ui/src/stores/useStore.ts b/ui/src/stores/useStore.ts index df598bf2f..6d497249e 100644 --- a/ui/src/stores/useStore.ts +++ b/ui/src/stores/useStore.ts @@ -36,7 +36,7 @@ import { type GenerationSubmissionContext, } from '../features/studio/generationProvenance' import { storyDirectorSubmissionProvenance } from '../features/stories/provenance' -import type { ImageGenerationReceipt } from '../api/imageGenerationCommands' +import type { GenerationReceiptLike } from '../api/generationCommandClient' import { prepareStudioSubmission, studioUploadReference } from '../features/studio/studioSubmission' const DASHBOARD_PIPELINE_PAGE_SIZE = 8 @@ -1520,7 +1520,7 @@ export interface AppState extends LlmSlice, StudioConfigurationSlice { startGeneration: ( scheduledPrompt?: ScheduledPromptSubmission, submissionContext?: GenerationSubmissionContext, - ) => Promise + ) => Promise stopGeneration: (jobId?: string) => void dismissJob: (jobId: string) => void reconnectJobs: () => Promise diff --git a/ui/tests/studioCapabilities.test.mjs b/ui/tests/studioCapabilities.test.mjs index 40c733363..816ca960f 100644 --- a/ui/tests/studioCapabilities.test.mjs +++ b/ui/tests/studioCapabilities.test.mjs @@ -145,3 +145,41 @@ test('image prompts are accepted intact or rejected rather than silently truncat assert.equal(definition.resolve({ type: 'prepare_image', prompt: 'x'.repeat(200_001) }), null) assert.equal(definition.resolve({ type: 'prepare_image', prompt, negative_prompt: 'x'.repeat(200_001) }), null) }) + +test('speech preparation preserves authored multiline text, language metadata and native duration', async () => { + const definitions = await registeredStudioCapabilities() + const definition = definitions.get('prepare_audio') + const prompt = ' Buenos días, Tentri.\nLee este diagnóstico exactamente. ' + const negative = ' No añadas una despedida.\nConserva los saltos. ' + const languageIntent = { + contentLanguage: 'es', technicalPromptLanguage: 'es', + verbatimSegments: [{ kind: 'spoken_text', text: prompt, language: 'es' }], + } + + const parsedAtZero = definition.resolve({ type: 'prepare_audio', audio_sub_mode: 'speech', + prompt, negative_prompt: negative, model_type: 'kugelaudio_0_open', duration_seconds: 0 }) + assert.equal(parsedAtZero.prompt, prompt) + assert.equal(parsedAtZero.negativePrompt, negative) + assert.equal(parsedAtZero.durationSeconds, 0) + + const parsedAtSixty = definition.resolve({ type: 'prepare_audio', audio_sub_mode: 'speech', + prompt, model_type: 'kugelaudio_0_open', duration_seconds: 60 }) + assert.equal(parsedAtSixty.durationSeconds, 60) + + const prepared = await definition.prepare({ ...parsedAtSixty, languageIntent }) + assert.equal(prepared.prompt, prompt) + assert.equal(prepared.negativePrompt, undefined) + assert.deepEqual(prepared.languageIntent, languageIntent) + + let received + await definition.execute(prepared, { adapters: { studio: { async prepareAudio(action) { + received = action + return { message: 'Prepared speech' } + } } } }) + assert.equal(received.prompt, prompt) + assert.deepEqual(received.languageIntent, languageIntent) + assert.equal(definition.resolve({ type: 'prepare_audio', audio_sub_mode: 'speech', + prompt: 'x'.repeat(200_001) }), null) + assert.equal(definition.resolve({ type: 'prepare_audio', audio_sub_mode: 'speech', + prompt, negative_prompt: 'x'.repeat(200_001) }), null) +}) diff --git a/ui/tests/studioSpeechCommandPresentation.test.tsx b/ui/tests/studioSpeechCommandPresentation.test.tsx new file mode 100644 index 000000000..2219d92e3 --- /dev/null +++ b/ui/tests/studioSpeechCommandPresentation.test.tsx @@ -0,0 +1,316 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import React from 'react' +import { JSDOM } from 'jsdom' +import type { SpeechPresentation } from '../src/features/studio/speechCommandPresentation' + +const dom = new JSDOM('', { url: 'http://localhost/' }) +let nextFrameId = 1 +const frameCallbacks = new Map() +const requestAnimationFrame = (callback: FrameRequestCallback): number => { + const id = nextFrameId++ + frameCallbacks.set(id, callback) + return id +} +const cancelAnimationFrame = (id: number): void => { frameCallbacks.delete(id) } + +Object.assign(globalThis, { + window: dom.window, + document: dom.window.document, + HTMLElement: dom.window.HTMLElement, + Event: dom.window.Event, + CustomEvent: dom.window.CustomEvent, + MutationObserver: dom.window.MutationObserver, + localStorage: dom.window.localStorage, + React, + requestAnimationFrame, + cancelAnimationFrame, + IS_REACT_ACT_ENVIRONMENT: true, +}) +Object.defineProperty(dom.window, 'requestAnimationFrame', { configurable: true, value: requestAnimationFrame }) +Object.defineProperty(dom.window, 'cancelAnimationFrame', { configurable: true, value: cancelAnimationFrame }) +Object.defineProperty(dom.window.HTMLElement.prototype, 'scrollIntoView', { configurable: true, value: () => undefined }) +Object.defineProperty(dom.window, 'setTimeout', { configurable: true, value: globalThis.setTimeout }) +Object.defineProperty(dom.window, 'clearTimeout', { configurable: true, value: globalThis.clearTimeout }) + +const { setUiLanguage } = await import('../src/i18n/index.ts') +await setUiLanguage('en') +const { StudioSpeechCommandPanel } = await import('../src/features/studio/StudioSpeechCommandPanel.tsx') +const { prepareStudioSpeechSubmission } = await import('../src/features/studio/speechCommandSubmission.ts') +const { createStudioSpeechGenerationCommand } = await import('../src/features/studio/speechGenerationSpec.ts') +const { pendingSpeechGenerationCommands } = await import('../src/api/speechGenerationCommands.ts') +const { presentStudioSpeechCommand, SPEECH_PRESENTATION_EVENT } = await import('../src/features/studio/speechCommandPresentation.ts') + +const originalFetch = globalThis.fetch + +async function flushAnimationFrames(): Promise { + for (let pass = 0; pass < 8 && frameCallbacks.size > 0; pass += 1) { + const pending = [...frameCallbacks.entries()] + frameCallbacks.clear() + for (const [, callback] of pending) callback(0) + await Promise.resolve() + } +} + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }) +} + +function baseParams(intent: string): Record { + return { + workspace: 'speech-ack-workspace', + prompt: `Speaker one says hello ${intent}\nsecond line stays exact`, + _tts_original_prompt: `Speaker one says hello ${intent}\nsecond line stays exact`, + model_type: 'kugelaudio_0_open', + resolution: '1280x720', + num_inference_steps: 0, + guidance_scale: 3, + seed: -1, + image_mode: 0, + video_length: 0, + generation_mode: 'audio', + _audio_sub_mode: 'speech', + negative_prompt: '', + repeat_generation: 1, + activated_loras: [], + loras_multipliers: '', + multi_prompts_gen_type: 2, + _tts_speaker_name1: '', + _tts_speaker_name2: '', + _tts_voice_count: 0, + duration_seconds: 20, + } +} + +function formState(params: Record) { + return { + params, + activeWorkspace: params.workspace, + generationMode: 'audio', + audioSubMode: 'speech', + durationSeconds: 20, + ttsVoiceCount: 0, + ttsSpeakerName1: '', + ttsSpeakerName2: '', + ttsVoices: [], + settingsOpen: false, + dashboardOpen: false, + sidebarMode: 'studio', + } as Parameters[1] +} + +function queuedResponse(body: Record): Response { + const input = body.input as Record + const workspace = String(input.workspace) + const intent = String(body.intent_id) + const taskId = `task-${intent}` + return jsonResponse({ + receipt: { + version: 1, + commandId: intent, + operation: 'generation.speech', + status: 'queued', + entities: [], + artifacts: [], + taskIds: [taskId], + pipelineIds: [], + result: { job_id: `job-${intent}`, task_id: taskId, workspace, status: 'queued' }, + commandVersion: 2, + fingerprintVersion: 2, + contentFingerprint: 'a'.repeat(64), + }, + replayed: false, + }) +} + +test.afterEach(() => { + frameCallbacks.clear() + dom.window.localStorage.clear() + globalThis.fetch = originalFetch + document.body.replaceChildren() +}) + +test('speech panel ACK shows original text, duration, voice and reference counts before POST', { concurrency: false }, async () => { + const { render, screen, waitFor, cleanup, act } = await import('@testing-library/react') + const params = baseParams('ack') + const state = formState(params) + const calls: Array<{ url: string; body?: Record }> = [] + globalThis.fetch = (async (input, init) => { + const url = String(input) + const body = init?.body ? JSON.parse(String(init.body)) as Record : undefined + calls.push({ url, body }) + if (url.includes('/references')) return jsonResponse({ references: body?.references || [] }) + if (url.endsWith('/generation/commands')) return queuedResponse(body || {}) + throw new Error(`unexpected fetch: ${url}`) + }) as typeof fetch + render( undefined} />) + try { + const prepared = await prepareStudioSpeechSubmission(params, state, () => state, { actor: 'wizard', commandId: 'speech-panel-ack' }) + let submission!: ReturnType + await act(async () => { + submission = prepared.submit() + await Promise.resolve() + }) + await waitFor(() => { + assert.match(screen.getByRole('status').textContent || '', /Speaker one says hello ack/) + assert.match(screen.getByRole('status').textContent || '', /20s/) + assert.match(screen.getByRole('status').textContent || '', /0 audio references · 0 voices · 0 LoRAs/) + }) + assert.equal(calls.filter(call => call.url.endsWith('/generation/commands')).length, 0) + await act(async () => { await flushAnimationFrames() }) + await submission + assert.equal(calls.filter(call => call.url.endsWith('/generation/commands')).length, 1) + const posted = calls.find(call => call.url.endsWith('/generation/commands'))?.body + assert.equal(posted?.operation, 'generation.speech') + assert.equal(((posted?.input as Record).params as Record)._tts_original_prompt, params._tts_original_prompt) + } finally { + cleanup() + } +}) + +test('changing duration or voice file while audio references resolve aborts before POST', { concurrency: false }, async () => { + const params = { ...baseParams('race'), audio_guide: '/api/v1/uploads/voice.wav' } + const before = formState(params) + let live = before + let resolveReference!: (value: Response) => void + let generationCalls = 0 + globalThis.fetch = (async (input) => { + const url = String(input) + if (url.includes('/references')) { + return await new Promise(resolve => { resolveReference = resolve }) + } + if (url.endsWith('/generation/commands')) generationCalls += 1 + return jsonResponse({}) + }) as typeof fetch + + const preparedPromise = prepareStudioSpeechSubmission(params, before, () => live, { actor: 'wizard', commandId: 'speech-race' }) + live = { + ...before, + durationSeconds: 28, + ttsVoiceCount: 1, + ttsVoices: [{ name: 'Voice', filename: 'voice.wav', path: '/workspace/voice.wav' }], + } + resolveReference(jsonResponse({ references: ['/api/v1/uploads/voice.wav'] })) + const prepared = await preparedPromise + await assert.rejects(prepared.submit(), /changed before submission/i) + assert.equal(generationCalls, 0) + assert.deepEqual(pendingSpeechGenerationCommands(), []) +}) + +test('a malformed audio reference is rejected before resolution and cannot shift a later voice slot', { concurrency: false }, async () => { + const params = { + ...baseParams('malformed-reference'), + audio_guide: 123, + audio_guide2: '/api/v1/uploads/later-voice.wav', + } + const before = formState(params) + let referenceCalls = 0 + let generationCalls = 0 + globalThis.fetch = (async input => { + if (String(input).includes('/references')) referenceCalls += 1 + if (String(input).endsWith('/generation/commands')) generationCalls += 1 + return jsonResponse({ references: ['/api/v1/uploads/resolved.wav'] }) + }) as typeof fetch + + const prepared = await prepareStudioSpeechSubmission( + params, + before, + () => before, + { actor: 'wizard', commandId: 'speech-malformed-reference' }, + ) + await assert.rejects(prepared.submit(), /audio_guide.*string audio reference/i) + assert.equal(referenceCalls, 0) + assert.equal(generationCalls, 0) +}) + +test('the native zero duration sentinel is presented as auto duration', { concurrency: false }, async () => { + const { render, screen, waitFor, cleanup, act } = await import('@testing-library/react') + const params = { ...baseParams('auto-duration'), duration_seconds: 0 } + const command = createStudioSpeechGenerationCommand(params, 'speech-auto-duration') + render( undefined} />) + try { + const presented = presentStudioSpeechCommand(command) + await waitFor(() => assert.match(screen.getByRole('status').textContent || '', /auto duration/i)) + await act(async () => { await flushAnimationFrames() }) + await presented + } finally { + cleanup() + } +}) + +test('Load Settings/reroll speech preparation projects stale H3 controls before the strict builder', { concurrency: false }, async () => { + const params = { + ...baseParams('reroll-load-settings'), + // These values can be restored into the shared form by a video/H3 + // sidecar. Speech keeps its native fields while dropping this known UI + // residue before entering the closed generation command. + h3_reference_mode: 'first_frame', + minimax_h3_planning_style: 'faithful', + minimax_h3_audio_policy: 'native', + minimax_h3_reference_sequence: false, + minimax_h3_turbo_preset: 'standard', + perturbation_layers: [9], + stage2_steps: 8, + voice_clone_enabled: true, + voice_clone_refs: ['/tmp/voice.wav'], + h3_ref_videos: [], + h3_ref_audios: [], + minimax_h3_references: [], + } + const state = formState(params) + const prepared = await prepareStudioSpeechSubmission( + params, + state, + () => state, + { actor: 'wizard', commandId: 'speech-reroll-load-settings' }, + ) + assert.equal(prepared.params.duration_seconds, 20) + assert.deepEqual(prepared.params.h3_ref_videos, []) + assert.deepEqual(prepared.params.h3_ref_audios, []) + assert.equal(prepared.params.minimax_h3_planning_style, undefined) + assert.equal(prepared.params.perturbation_layers, undefined) + assert.equal(prepared.params.voice_clone_refs, undefined) +}) + +test('speech presentation waits for the listening panel and cancels when it is removed', { concurrency: false }, async () => { + const command = createStudioSpeechGenerationCommand(baseParams('lifecycle'), 'speech-lifecycle') + const root = document.createElement('div') + root.dataset.studioSpeechReady = 'true' + root.dataset.studioSpeechListening = 'false' + document.body.append(root) + const requests: SpeechPresentation[] = [] + const receive = (event: Event) => { + const request = (event as CustomEvent).detail + requests.push(request) + root.dataset.studioSpeechCommand = request.command.intent_id + request.respond() + } + window.addEventListener(SPEECH_PRESENTATION_EVENT, receive) + const waiting = presentStudioSpeechCommand(command) + await new Promise(resolve => setTimeout(resolve, 0)) + assert.equal(requests.length, 0) + root.dataset.studioSpeechListening = 'true' + await waiting + assert.equal(requests.length, 1) + window.removeEventListener(SPEECH_PRESENTATION_EVENT, receive) + root.remove() + + const disappearing = document.createElement('div') + disappearing.dataset.studioSpeechReady = 'true' + disappearing.dataset.studioSpeechListening = 'true' + document.body.append(disappearing) + const removeOnRequest = (event: Event) => { + const request = (event as CustomEvent).detail + disappearing.remove() + assert.equal(request.active, true) + } + window.addEventListener(SPEECH_PRESENTATION_EVENT, removeOnRequest) + await assert.rejects( + presentStudioSpeechCommand(createStudioSpeechGenerationCommand(baseParams('removed'), 'speech-removed')), + /could not be shown/i, + ) + window.removeEventListener(SPEECH_PRESENTATION_EVENT, removeOnRequest) +}) diff --git a/ui/tests/studioSpeechGenerationCommands.test.ts b/ui/tests/studioSpeechGenerationCommands.test.ts new file mode 100644 index 000000000..b357de88c --- /dev/null +++ b/ui/tests/studioSpeechGenerationCommands.test.ts @@ -0,0 +1,235 @@ +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import test from 'node:test' +import { JSDOM } from 'jsdom' + +const dom = new JSDOM('', { url: 'http://localhost/' }) +Object.assign(globalThis, { + window: dom.window, + document: dom.window.document, + Event: dom.window.Event, + localStorage: dom.window.localStorage, +}) + +const { + createStudioSpeechGenerationCommand, + fetchSpeechGenerationCommandReceipt, + pendingSpeechGenerationCommands, + submitSpeechGenerationCommand, + SpeechGenerationCommandError, +} = await import('../src/api/speechGenerationCommands.ts') +const { + STUDIO_SPEECH_PARAM_KEYS, + projectStudioSpeechFormParams: projectSpeechFormParams, +} = await import('../src/features/studio/speechGenerationSpec.ts') + +const originalFetch = globalThis.fetch + +function response(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }) +} + +function nativeParams(): Record { + return JSON.parse(readFileSync( + new URL('../../tests/fixtures/studio_speech_native_request.json', import.meta.url), + 'utf8', + )) as Record +} + +function command(intentId = 'speech-ui-intent'): ReturnType { + const params = nativeParams() + const workspace = params.workspace + delete params.workspace + return createStudioSpeechGenerationCommand({ + ...params, + workspace, + provenance: { actor: 'wizard', workspace_id: 'speech-collection' }, + }, intentId) +} + +function queuedReceipt(value: { intent_id: string; operation: string; input: { workspace: string } }) { + return { + receipt: { + version: 1, + commandId: value.intent_id, + operation: value.operation, + status: 'queued', + entities: [], + artifacts: [], + taskIds: [`task-${value.intent_id}`], + pipelineIds: [], + result: { + job_id: `job-${value.intent_id}`, + task_id: `task-${value.intent_id}`, + workspace: value.input.workspace, + status: 'queued', + }, + commandVersion: 2, + fingerprintVersion: 2, + contentFingerprint: 'a'.repeat(64), + }, + replayed: false, + } +} + +function bodyOf(init: RequestInit | undefined): Record { + assert.ok(init?.body) + return JSON.parse(String(init.body)) as Record +} + +test.afterEach(() => { + dom.window.localStorage.clear() + globalThis.fetch = originalFetch +}) + +test('speech command is a detached closed catalog snapshot with original TTS text', { concurrency: false }, () => { + const params = nativeParams() + const workspace = params.workspace + delete params.workspace + const command = createStudioSpeechGenerationCommand({ + ...params, + workspace, + provenance: { actor: 'wizard', workspace_id: 'speech-collection' }, + }, ' speech-intent ') + + assert.equal(command.version, 2) + assert.equal(command.operation, 'generation.speech') + assert.equal(command.input.workspace, 'speech-test') + assert.equal(command.input.workspace_collection_id, 'speech-collection') + assert.equal(command.input.params._audio_sub_mode, 'speech') + assert.equal(command.input.params._tts_original_prompt, 'The system is watching.\nEvery warning matters.') + assert.equal(command.input.params.duration_seconds, 20) + assert.ok(STUDIO_SPEECH_PARAM_KEYS.includes('duration_seconds')) + assert.equal('workspace' in command.input.params, false) + + params.prompt = 'changed after freeze' + assert.equal(command.input.params.prompt, 'The system is watching.\nEvery warning matters.') + + assert.throws( + () => createStudioSpeechGenerationCommand({ ...nativeParams(), workspace: 'speech-test', unknown_native_key: true }, 'bad'), + /unknown_native_key/, + ) + assert.throws( + () => createStudioSpeechGenerationCommand({ ...nativeParams(), workspace: 'speech-test', audio_guide: '/home/ina/private.wav' }, 'path'), + /audio_guide/, + ) + assert.throws( + () => createStudioSpeechGenerationCommand({ ...nativeParams(), workspace: 'speech-test', generation_mode: 'image' }, 'mode'), + /generation_mode/, + ) +}) + +test('Load Settings/reroll projects known video and H3 form residue without weakening the closed builder', { concurrency: false }, () => { + const source = { + ...nativeParams(), + workspace: 'speech-test', + provenance: { actor: 'wizard', workspace_id: 'speech-collection' }, + // Native inactive values are retained in the command snapshot. + h3_ref_videos: [], + h3_ref_audios: [], + minimax_h3_references: [], + minimax_h3_turbo_mode: false, + // These are restored by Load Settings from a video/H3 sidecar, but are + // UI controls outside the published speech parameter catalog. + video_prompt_type: '', + image_prompt_type: '', + minimax_h3_planning_style: 'faithful', + minimax_h3_audio_policy: 'native', + minimax_h3_reference_sequence: false, + minimax_h3_turbo_preset: 'standard', + h3_reference_mode: 'first_frame', + perturbation_switch: 0, + perturbation_layers: [9], + stg_scale: 1, + voice_clone_enabled: true, + voice_clone_refs: ['/tmp/private-voice.wav'], + } as Record + + const projection = projectSpeechFormParams(source) + assert.deepEqual(projection.params.h3_ref_videos, []) + assert.deepEqual(projection.params.h3_ref_audios, []) + assert.deepEqual(projection.params.minimax_h3_references, []) + assert.equal(projection.params.minimax_h3_turbo_mode, false) + assert.equal(projection.params.duration_seconds, 20) + assert.ok(projection.droppedFields.includes('minimax_h3_planning_style')) + assert.ok(projection.droppedFields.includes('perturbation_layers')) + assert.ok(projection.droppedFields.includes('voice_clone_refs')) + assert.equal('minimax_h3_planning_style' in projection.params, false) + assert.equal('perturbation_layers' in projection.params, false) + assert.equal('voice_clone_refs' in projection.params, false) + + const projectedCommand = createStudioSpeechGenerationCommand(projection.params, 'speech-reroll') + assert.equal(projectedCommand.input.params._tts_original_prompt, source._tts_original_prompt) + + // A direct command/MCP caller still cannot use the form projection as an + // escape hatch: the closed builder rejects the same unknown field. + assert.throws( + () => createStudioSpeechGenerationCommand( + { ...projection.params, minimax_h3_planning_style: 'faithful' }, + 'speech-direct-unknown', + ), + /minimax_h3_planning_style/, + ) +}) + +test('speech admission persists exact envelope before POST and validates typed v2 receipt', { concurrency: false }, async () => { + const value = command('speech-submit') + const calls: Array<{ url: string; init?: RequestInit }> = [] + globalThis.fetch = async (url, init) => { + calls.push({ url: String(url), init }) + return response(queuedReceipt(value)) + } + let sawPending = false + const receipt = await submitSpeechGenerationCommand(value, { + submissionContext: { actor: 'wizard', workflowId: 'speech-workflow' }, + onSnapshotReady: snapshot => { + sawPending = pendingSpeechGenerationCommands('speech-test').length === 1 + assert.deepEqual(snapshot, value) + }, + }) + assert.equal(sawPending, true) + assert.equal(calls.length, 1) + assert.equal(calls[0]?.url.endsWith('/api/v1/generation/commands'), true) + assert.deepEqual(bodyOf(calls[0]?.init), value) + assert.equal((calls[0]?.init?.headers as Record)['X-Hocus-UI-Surface'], 'wizard') + assert.equal(receipt.operation, 'generation.speech') + assert.equal(receipt.result.job_id, 'job-speech-submit') + assert.deepEqual(pendingSpeechGenerationCommands(), []) +}) + +test('an uncertain speech response retries the same intent and command', { concurrency: false }, async () => { + const value = command('speech-retry') + const bodies: Record[] = [] + let attempt = 0 + globalThis.fetch = async (_url, init) => { + bodies.push(bodyOf(init)) + attempt += 1 + if (attempt === 1) throw new Error('connection lost after admission') + return response(queuedReceipt(value)) + } + await assert.rejects( + submitSpeechGenerationCommand(value), + error => error instanceof SpeechGenerationCommandError && error.uncertain, + ) + assert.deepEqual(pendingSpeechGenerationCommands(), [value]) + const recovered = await submitSpeechGenerationCommand(value) + assert.equal(recovered.result.task_id, 'task-speech-retry') + assert.deepEqual(bodies, [value, value]) + assert.deepEqual(pendingSpeechGenerationCommands(), []) +}) + +test('receipt recovery retains speech v2 fingerprint requirements', { concurrency: false }, async () => { + const value = command('speech-receipt') + globalThis.fetch = async () => { throw new Error('connection lost') } + await assert.rejects(submitSpeechGenerationCommand(value), /connection lost/) + globalThis.fetch = async (url) => { + assert.match(String(url), /receipt\?workspace=speech-test&intent_id=speech-receipt/) + return response(queuedReceipt(value)) + } + const recovered = await fetchSpeechGenerationCommandReceipt('speech-test', value.intent_id) + assert.equal(recovered.operation, 'generation.speech') + assert.deepEqual(pendingSpeechGenerationCommands(), []) +}) From 888928614b6ce77460010fead1c47f29b1f22705 Mon Sep 17 00:00:00 2001 From: IAnMove <216241348+IAnMove@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:07:56 +0200 Subject: [PATCH 03/59] feat: add shared tools upscale command contract --- app/routers/tools_upscale_commands.py | 51 +++ app/services/tools_upscale_commands.py | 197 +++++++++++ app/services/tools_upscale_preparation.py | 399 ++++++++++++++++++++++ app/services/tools_upscale_spec.py | 330 ++++++++++++++++++ docs/development/TOOLS_COMMANDS.md | 64 ++++ tests/test_tools_upscale_preparation.py | 282 +++++++++++++++ tests/test_tools_upscale_spec.py | 149 ++++++++ 7 files changed, 1472 insertions(+) create mode 100644 app/routers/tools_upscale_commands.py create mode 100644 app/services/tools_upscale_commands.py create mode 100644 app/services/tools_upscale_preparation.py create mode 100644 app/services/tools_upscale_spec.py create mode 100644 docs/development/TOOLS_COMMANDS.md create mode 100644 tests/test_tools_upscale_preparation.py create mode 100644 tests/test_tools_upscale_spec.py diff --git a/app/routers/tools_upscale_commands.py b/app/routers/tools_upscale_commands.py new file mode 100644 index 000000000..fbcf69385 --- /dev/null +++ b/app/routers/tools_upscale_commands.py @@ -0,0 +1,51 @@ +"""Pure catalog projection for the typed Tools upscale operation.""" + +from __future__ import annotations + +from copy import deepcopy + +from services.tools_upscale_spec import tools_upscale_schema + + +def tools_upscale_command_catalog() -> dict: + """Return the exact operation entry consumed by HTTP/MCP discovery.""" + schema = tools_upscale_schema() + input_schema = dict(schema["input"]) + definitions = input_schema.pop("$defs", {}) + return { + "name": "tools.upscale", + "version": 2, + "supportedVersions": [2], + "domain": "tools", + "mutation": True, + "description": ( + "Upscale one exact image or video source in an explicit output " + "workspace with an installed local processor. Preserve the " + "source workspace, processor settings and intent_id; the shared " + "receipt proves admission and its canonical task reports completion." + ), + "inputSchema": { + "type": "object", + "additionalProperties": False, + "$defs": definitions, + "properties": { + "version": {"type": "integer", "const": 2}, + "operation": {"const": "tools.upscale"}, + "intent_id": deepcopy(schema["intent_id"]), + "input": input_schema, + }, + "required": ["version", "operation", "intent_id", "input"], + }, + } + + +# Short aliases used by catalog exporters in adjacent command slices. +tools_upscale_catalog = tools_upscale_command_catalog +upscale_command_catalog = tools_upscale_command_catalog + + +__all__ = [ + "tools_upscale_catalog", + "tools_upscale_command_catalog", + "upscale_command_catalog", +] diff --git a/app/services/tools_upscale_commands.py b/app/services/tools_upscale_commands.py new file mode 100644 index 000000000..8dbb63927 --- /dev/null +++ b/app/services/tools_upscale_commands.py @@ -0,0 +1,197 @@ +"""Bind the typed Tools upscale contract to the shared native adapter. + +This module is deliberately an adapter only. Queue ownership, durable +receipts and worker dispatch remain in ``ImageGenerationCommands`` and the +existing launch runtime. In particular, this file does not define a second +registry or start a thread. +""" + +from __future__ import annotations + +from copy import deepcopy +import os +from typing import Any + +from services.image_generation_commands import command_error +from services.native_generation_operation import NativeGenerationOperation +from services.tools_upscale_preparation import prepare_tools_upscale +from services.tools_upscale_spec import ( + _asset_id_from_reference, + freeze_tools_upscale_spec, +) + + +def _execution_policy(runtime): + execution_mode = runtime.get("execution_mode") + validate = getattr(execution_mode, "validate_generation", None) + if not callable(validate): + return lambda _workspace: None + + def check(workspace): + try: + validate(workspace) + except Exception as error: + error_type = getattr(execution_mode, "ExecutionModeError", ValueError) + if isinstance(error, error_type): + raise command_error(409, "execution_policy", str(error)) from error + raise + + return check + + +def _resource_adapter(runtime): + from services.studio_image_resources import StudioImageResources + + workspace_dir = runtime.get("_workspace_dir") + list_workspaces = runtime.get("_list_workspaces") + if not callable(workspace_dir) or not callable(list_workspaces): + raise ValueError("Tools upscale resources are not configured") + uploads_dir = runtime.get("_uploads_dir") + if not callable(uploads_dir): + uploads_dir = lambda: os.path.join(os.getcwd(), "uploads") + wgp = runtime.get("wgp") + lora_search_dirs = getattr(wgp, "get_lora_search_dirs", None) + if not callable(lora_search_dirs): + lora_search_dirs = lambda _model_type: [] + lora_compatible = runtime.get("_lora_is_compatible_with_model") + if not callable(lora_compatible): + lora_compatible = lambda _definition, _path: True + return StudioImageResources( + workspace_dir=workspace_dir, + uploads_dir=uploads_dir, + list_workspaces=list_workspaces, + lora_search_dirs=lora_search_dirs, + lora_compatible=lora_compatible, + ) + + +def _resolve_source(runtime): + resolver = runtime.get("_resolve_tool_source") or runtime.get("resolve_tool_source") + if not callable(resolver): + return None + + def resolve(params, **kwargs): + # Preparation already adapts bare/API asset references to the native + # ``asset_id`` field. Accept that prepared body as-is while also + # supporting direct adapter callers that still provide ``source``. + body = deepcopy(params) + source = body.get("source") + asset_id = body.get("asset_id") + if asset_id is None and isinstance(source, str): + asset_id = _asset_id_from_reference(source) + if asset_id: + body["asset_id"] = asset_id + body.pop("source", None) + expected_kinds = kwargs.get("expected_kinds") or (body.get("source_kind"),) + try: + return resolver(body, expected_kinds=expected_kinds) + except TypeError as first_error: + try: + return resolver(body) + except TypeError: + raise first_error + + return resolve + + +def _request_ready_params(params: dict[str, Any]) -> dict[str, Any]: + """Project inspected source identity into the legacy endpoint body. + + The endpoint accepts one source field. The typed adapter has already + inspected the canonical URL/asset, so it sends the exact resolved path + through the existing confined resolver until a canonical API URL can be + projected. A submitted managed asset ID is retained alongside that URL; + the native resolver checks that both identify the same basename/location. + """ + result = deepcopy(params) + source = result.pop("source", None) + asset_id = _asset_id_from_reference(source) if isinstance(source, str) else None + if asset_id: + result["asset_id"] = asset_id + path_key = "source_path" if result.get("source_kind") == "image" else "video_path" + result[path_key] = result.get(path_key) + return result + + +def _canonical_request_source(params: dict[str, Any], runtime: dict[str, Any]) -> str | None: + path_key = "source_path" if params.get("source_kind") == "image" else "video_path" + path = params.get(path_key) + workspace_dir = runtime.get("_workspace_dir") + if not path or not callable(workspace_dir): + return None + uploads_dir = runtime.get("_uploads_dir") + if not callable(uploads_dir): + uploads_dir = lambda: os.path.join(os.getcwd(), "uploads") + source_workspace = params.get("source_workspace") + if not isinstance(source_workspace, str) or not source_workspace: + return None + try: + from services.wangp_submission import wangp_media_url + + workspace_for_url = params.get("workspace") if source_workspace == "__uploads__" else source_workspace + return wangp_media_url( + path, + workspace_for_url, + uploads_dir=uploads_dir(), + workspace_dir=workspace_dir( + source_workspace if source_workspace != "__uploads__" else params.get("workspace") + ), + ) + except (OSError, TypeError, ValueError): + return None + + +def _worker(runtime): + policy = runtime["execution_mode"].policy() + return runtime["_run_generation"] if policy.simulated else runtime["_run_tool_upscale"] + + +def create_tools_upscale_operation(runtime: dict[str, Any]) -> NativeGenerationOperation: + """Create the registered ``tools.upscale`` adapter for one runtime.""" + source_resolver = _resolve_source(runtime) + + def freeze(command): + frozen = freeze_tools_upscale_spec(command) + effective = frozen["effective"]["input"] + return frozen, { + **deepcopy(effective["params"]), + "workspace": effective["workspace"], + } + + def prepare(params): + prepared, resources = prepare_tools_upscale( + params, + resources=_resource_adapter(runtime), + resolve_source=source_resolver, + execution_policy=_execution_policy(runtime), + processor_capabilities=runtime.get("_tools_processor_capabilities"), + validate_processors=runtime.get("_validate_tools_processors"), + processor_settings=runtime.get("_validated_tools_processor_settings"), + processor_parameters=runtime.get("_tools_processor_parameters"), + probe_video=runtime.get("_probe_tool_video"), + ) + ready = _request_ready_params(prepared) + canonical_source = _canonical_request_source(prepared, runtime) + if canonical_source: + path_key = "source_path" if prepared.get("source_kind") == "image" else "video_path" + ready.pop(path_key, None) + ready["source"] = canonical_source + return ready, resources + + from routers.tools_upscale_commands import tools_upscale_command_catalog + + return NativeGenerationOperation( + freeze=freeze, + prepare=prepare, + catalog=tools_upscale_command_catalog(), + prepare_request=runtime["tools_upscale"], + worker=_worker(runtime), + use_generation_defaults=False, + ) + + +# Alias kept for discovery code that calls adapters by their operation name. +create_tools_upscale_adapter = create_tools_upscale_operation + + +__all__ = ["create_tools_upscale_adapter", "create_tools_upscale_operation"] diff --git a/app/services/tools_upscale_preparation.py b/app/services/tools_upscale_preparation.py new file mode 100644 index 000000000..ff47a87f1 --- /dev/null +++ b/app/services/tools_upscale_preparation.py @@ -0,0 +1,399 @@ +"""Provider-free source and processor preparation for ``tools.upscale``. + +The preparation boundary runs after the typed command has been frozen and +before the shared admission callback is invoked. It uses the existing Tools +source resolver and WangGP processor validators, then returns a detached +native worker snapshot plus portable resource identities. It never starts a +worker, calls an endpoint, downloads a model or writes an output. +""" + +from __future__ import annotations + +from copy import deepcopy +from collections.abc import Callable, Mapping +import os +from pathlib import Path +from typing import Any + +from fastapi import HTTPException +from pydantic import ValidationError + +from services.image_generation_commands import command_error +from services.studio_image_resources import file_identity +from services.tools_upscale import TOOL_SOURCE_EXTENSIONS +from services.tools_upscale_spec import ToolsUpscaleParams + + +def _processor_defaults() -> tuple[Callable, Callable, Callable]: + from shared.wangp1272 import processors + + return processors.capabilities, processors.validate_selection, processors.validated_settings + + +def _validated_params(params: Any) -> tuple[dict[str, Any], str]: + if type(params) is not dict: + raise command_error(422, "invalid_tools_upscale_input", "Tools upscale parameters must be an object") + workspace = params.get("workspace") + if not isinstance(workspace, str) or not workspace.strip(): + raise command_error(422, "invalid_workspace", "Use an explicit output workspace") + try: + parsed = ToolsUpscaleParams.model_validate( + {key: value for key, value in params.items() if key != "workspace"} + ) + except ValidationError as error: + details = "; ".join( + f"{'.'.join(str(part) for part in item.get('loc', ())) or 'input.params'}: " + f"{item.get('msg') or 'Invalid value'}" + for item in error.errors(include_input=False) + ) + raise command_error(422, "invalid_tools_upscale_input", details) from error + working = parsed.model_dump(mode="json") + working["wangp_processor_settings"] = working.get("wangp_processor_settings") or {} + working["workspace"] = workspace + return working, workspace + + +def _call_source_resolver(resolver: Callable, params: dict[str, Any], kind: str): + request = { + "source": params["source"], + "source_kind": kind, + "workspace": params["workspace"], + } + # The legacy runtime resolver accepts asset_id separately. Keep the + # typed contract's single canonical ``source`` field while adapting only + # this in-process call; the frozen receipt still contains the source ID. + source = str(params["source"]) + if source.startswith(("asset_", "asset:", "asset-")): + request["asset_id"] = source + request.pop("source", None) + elif source.startswith("/api/v1/assets/"): + from services.tools_upscale_spec import _asset_id_from_reference + + request["asset_id"] = _asset_id_from_reference(source) + request.pop("source", None) + try: + return resolver(request, expected_kinds=(kind,)) + except TypeError as first_error: + try: + return resolver(request) + except TypeError: + raise first_error + + +def _fallback_source(params: dict[str, Any], resources: Any): + if str(params.get("source_kind")) != "image" or not hasattr(resources, "_media"): + raise ValueError("A canonical Tools source resolver is required for this media kind") + path, source_workspace = resources._media(params["source"]) + from services.tools_upscale_spec import _asset_id_from_reference + + return { + "path": path, + "filename": os.path.basename(os.fspath(path)), + "source_workspace": source_workspace, + "source_kind": "image", + "asset_id": _asset_id_from_reference(str(params["source"])), + } + + +def _normalise_source(result: Any, expected_kind: str) -> dict[str, Any]: + if isinstance(result, Mapping): + values = { + "path": result.get("path") or result.get("source_path") or result.get("resolved"), + "filename": result.get("filename") or result.get("source_filename"), + "source_workspace": result.get("source_workspace") or result.get("workspace"), + "source_kind": result.get("source_kind") or result.get("kind"), + "asset_id": result.get("asset_id") or result.get("source_asset_id"), + "output_workspace": result.get("output_workspace"), + "output_dir": result.get("output_dir"), + } + elif isinstance(result, (tuple, list)) and len(result) >= 4: + values = { + "path": result[0], "filename": result[1], "source_workspace": result[2], + "source_kind": result[3], "asset_id": result[4] if len(result) > 4 else None, + "output_workspace": result[5] if len(result) > 5 else None, + "output_dir": result[6] if len(result) > 6 else None, + } + else: + raise ValueError("The Tools source resolver returned an invalid result") + path = values["path"] + if not isinstance(path, (str, os.PathLike)) or not os.fspath(path): + raise ValueError("The selected Tools source is unavailable") + path = os.fspath(path) + filename = values["filename"] or os.path.basename(path) + if not isinstance(filename, str) or os.path.basename(filename) != filename: + raise ValueError("The selected source filename is invalid") + source_workspace = values["source_workspace"] + if not isinstance(source_workspace, str) or not source_workspace.strip(): + raise ValueError("The selected source has no source workspace") + source_kind = values["source_kind"] or expected_kind + if source_kind != expected_kind: + raise ValueError("Source kind does not match the typed command") + return {**values, "path": path, "filename": filename, "source_kind": source_kind} + + +def _confined_to_resource_root(path: str, source_workspace: str, resources: Any) -> None: + root_factory = getattr(resources, "uploads_dir", None) if source_workspace == "__uploads__" else getattr(resources, "workspace_dir", None) + if not callable(root_factory): + return + try: + root = Path(root_factory() if source_workspace == "__uploads__" else root_factory(source_workspace)).resolve() + resolved = Path(path).resolve() + except (OSError, TypeError, ValueError) as error: + raise ValueError("The selected source location is unavailable") from error + if not resolved.is_relative_to(root) or resolved == root: + raise ValueError("The selected source is outside its declared workspace") + + +def _check_source_path(source: dict[str, Any], resources: Any) -> Path: + path = Path(source["path"]) + if not path.is_file(): + raise ValueError("The selected source is unavailable") + _confined_to_resource_root(os.fspath(path), source["source_workspace"], resources) + suffix = path.suffix.casefold() + if suffix not in TOOL_SOURCE_EXTENSIONS[source["source_kind"]]: + raise ValueError("Source kind does not match the file format") + return path + + +def _inspect_image(path: Path) -> dict[str, Any]: + from PIL import Image + + try: + with Image.open(path) as picture: + picture.verify() + with Image.open(path) as picture: + return {"format": picture.format, "width": picture.width, "height": picture.height} + except (OSError, SyntaxError, ValueError) as error: + raise ValueError("The selected image could not be decoded") from error + + +def _inspect_video(path: Path, probe_video: Callable | None) -> dict[str, Any]: + probe = probe_video + if probe is None: + from services.video_editor import probe_media + + probe = probe_media + try: + metadata = probe(os.fspath(path)) + except Exception as error: + raise ValueError("The selected video could not be inspected") from error + if not isinstance(metadata, Mapping): + raise ValueError("The video probe returned invalid metadata") + try: + duration = float(metadata.get("duration") or 0) + width = int(metadata.get("width") or 0) + height = int(metadata.get("height") or 0) + except (TypeError, ValueError) as error: + raise ValueError("The video probe returned invalid metadata") from error + import math + + if not math.isfinite(duration) or duration <= 0 or width <= 0 or height <= 0: + raise ValueError("The selected video has no readable duration or dimensions") + return deepcopy(dict(metadata)) + + +def _source_resource(source: dict[str, Any], path: Path, resources: Any, probe_video: Callable | None) -> dict[str, Any]: + identity = file_identity(path) + metadata = (_inspect_image(path) if source["source_kind"] == "image" + else _inspect_video(path, probe_video)) + return { + "role": "source", + "index": 0, + "url": source.get("url"), + "workspace": source["source_workspace"], + "kind": source["source_kind"], + "filename": source["filename"], + "asset_id": source.get("asset_id"), + "media": metadata, + **identity, + } + + +def _resolve_reference(resources: Any, value: str): + resolver = getattr(resources, "resolve_reference", None) + if callable(resolver): + try: + return resolver(value, media_kind="image") + except TypeError as first_error: + try: + return resolver(value) + except TypeError: + raise first_error + media = getattr(resources, "_media", None) + if callable(media): + return media(value) + raise ValueError("Processor references require the existing media resolver") + + +def _processor_reference_resources(settings: dict[str, Any], resources: Any) -> tuple[dict[str, Any], list[dict[str, Any]]]: + values = settings.get("spatial_upsampler_reference_images") or [] + if not values: + return settings, [] + prepared: list[str] = [] + identities: list[dict[str, Any]] = [] + for index, value in enumerate(values): + try: + path, workspace = _resolve_reference(resources, value) + except (OSError, TypeError, ValueError) as error: + raise ValueError(f"Processor reference {index} is unavailable") from error + path = Path(path) + if not path.is_file(): + raise ValueError(f"Processor reference {index} is unavailable") + _confined_to_resource_root(os.fspath(path), workspace, resources) + identities.append({ + "role": "processor_reference", "index": index, "url": value, + "workspace": workspace, **file_identity(path), "media": _inspect_image(path), + }) + prepared.append(os.fspath(path)) + result = deepcopy(settings) + result["spatial_upsampler_reference_images"] = prepared + return result, identities + + +def _declared_parameters(method: str, resolver: Callable | None = None) -> dict[str, dict[str, Any]]: + if resolver is not None: + try: + values = resolver(method) + except (ImportError, AttributeError, RuntimeError): + values = [] + else: + try: + from postprocessing.spatial_upsamplers import method_parameters + + values = method_parameters(method) + except (ImportError, AttributeError, RuntimeError): + values = [] + if isinstance(values, Mapping): + values = values.values() + if not isinstance(values, (list, tuple)): + values = [] + return { + str(item["name"]): item + for item in values + if isinstance(item, Mapping) and isinstance(item.get("name"), str) + } + + +def _validate_processor_settings(method: str, settings: dict[str, Any], validator: Callable, + parameter_resolver: Callable | None = None) -> dict[str, Any]: + submitted = {key: value for key, value in settings.items() if value is not None} + resolved = validator(method, submitted) + if not isinstance(resolved, dict): + raise ValueError("Processor settings validator returned an invalid object") + declared = _declared_parameters(method, parameter_resolver) + unsupported = set(submitted) - set(resolved) + unsupported -= { + key for key in unsupported + if declared.get(key, {}).get("type") == "array" + } + if unsupported: + raise ValueError("Processor settings contain values not declared by the selected processor") + result = deepcopy(resolved) + for key in set(submitted) - set(resolved): + if declared.get(key, {}).get("type") != "array": + continue + result[key] = deepcopy(submitted[key]) + return result + + +def _validate_processor(method: str, source_kind: str, capabilities: Callable | None, + selection_validator: Callable, settings_validator: Callable, + settings: dict[str, Any], parameter_resolver: Callable | None = None) -> dict[str, Any]: + temporal = method.startswith(("rife", "dlssg")) + spatial_value, temporal_value = ("", method) if temporal else (method, "") + error = selection_validator(spatial_value, temporal_value, source_kind == "image") + if error: + raise ValueError(str(error)) + if callable(capabilities): + records = capabilities() + if isinstance(records, (list, tuple)): + selected = next((item for item in records if isinstance(item, Mapping) and item.get("value") == method), None) + if selected is not None: + if selected.get("enabled") is False: + raise ValueError("The selected processor is disabled") + if source_kind not in selected.get("media", (source_kind,)): + raise ValueError("The selected processor does not support this source kind") + return _validate_processor_settings(method, settings, settings_validator, parameter_resolver) + + +def _native_snapshot(working: dict[str, Any], source: dict[str, Any], path: Path, + settings: dict[str, Any]) -> dict[str, Any]: + source_ref = { + "id": source.get("asset_id"), "kind": source["source_kind"], + "uri": source["filename"], "role": "source", + } + result = deepcopy(working) + result.update({ + "source_path": os.fspath(path) if source["source_kind"] == "image" else None, + "video_path": os.fspath(path) if source["source_kind"] == "video" else None, + "source_filename": source["filename"], + "source_workspace": source["source_workspace"], + "source_asset_id": source.get("asset_id"), + "model_type": "post_processing", + "generation_mode": source["source_kind"], + "provider": "local", + "capability": "tools.upscale", + "inputs": [source_ref], + "parents": [source_ref], + "transformations": [{ + "type": "upscale", "method": working["method"], + }], + "wangp_processor_settings": deepcopy(settings), + }) + return result + + +def prepare_tools_upscale( + params: dict[str, Any], *, resources: Any, resolve_source: Callable | None = None, + execution_policy: Callable | None = None, processor_capabilities: Callable | None = None, + validate_processors: Callable | None = None, processor_settings: Callable | None = None, + processor_parameters: Callable | None = None, probe_video: Callable | None = None, +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """Resolve and inspect one typed Tools source before shared admission.""" + working, workspace = _validated_params(params) + try: + if execution_policy is not None: + execution_policy(workspace) + defaults = None + if processor_capabilities is None or validate_processors is None or processor_settings is None: + defaults = _processor_defaults() + capabilities = processor_capabilities or defaults[0] + selection_validator = validate_processors or defaults[1] + settings_validator = processor_settings or defaults[2] + resolver = resolve_source or getattr(resources, "resolve_source", None) + kind = working["source_kind"] + result = (_call_source_resolver(resolver, working, kind) if callable(resolver) + else _fallback_source(working, resources)) + source = _normalise_source(result, kind) + source["url"] = working["source"] + path = _check_source_path(source, resources) + identities = [_source_resource(source, path, resources, probe_video)] + # Resolve processor reference media first. The native validator then + # sees the exact confined paths it will receive, while the frozen + # receipt retains submitted URLs and independent identities. + settings, processor_refs = _processor_reference_resources( + working["wangp_processor_settings"], resources + ) + settings = _validate_processor( + working["method"], kind, capabilities, selection_validator, + settings_validator, settings, processor_parameters, + ) + identities.extend(processor_refs) + return _native_snapshot(working, source, path, settings), deepcopy(identities) + except HTTPException: + raise + except (OSError, ValueError) as error: + raise command_error(422, "invalid_tools_upscale_input", str(error)) from error + + +# Descriptive aliases used by adapter discovery and tests in the other command +# slices. They all call this one preparation boundary. +prepare_tools_upscale_command = prepare_tools_upscale +prepare_tools_upscale_inputs = prepare_tools_upscale + + +__all__ = [ + "prepare_tools_upscale", + "prepare_tools_upscale_command", + "prepare_tools_upscale_inputs", +] diff --git a/app/services/tools_upscale_spec.py b/app/services/tools_upscale_spec.py new file mode 100644 index 000000000..617eeb2d6 --- /dev/null +++ b/app/services/tools_upscale_spec.py @@ -0,0 +1,330 @@ +"""Closed, provider-free command contract for ``tools.upscale``. + +The Tools worker already has a native implementation for spatial and temporal +upscaling. This module only freezes the small transport boundary that lets +Wizard, Studio and an external agent submit that implementation through the +shared admission path. It does not resolve a source, inspect a file, load a +processor or enqueue a job. + +The envelope is:: + + { + "version": 2, + "operation": "tools.upscale", + "intent_id": "...", + "input": { + "workspace": "...", + "workspace_collection_id": "...", + "params": { + "source": "asset_... or /api/v1/...", + "source_kind": "image" or "video", + "method": "lanczos2", + "seed": -1, + "wangp_processor_settings": { ... } + } + } + } + +``original`` is a detached copy of the submitted envelope. ``effective`` +adds only the deterministic seed/settings defaults and retains the source +spelling exactly. The fingerprint covers operation, workspace and effective +parameters, excluding the transport intent and caller metadata. +""" + +from __future__ import annotations + +from copy import deepcopy +import hashlib +import json +from typing import Annotated, Any, Literal +from urllib.parse import unquote, urlsplit + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StringConstraints, + StrictInt, + StrictStr, + ValidationError, + field_validator, + model_validator, +) + +from services.image_generation_spec import ImageGenerationSpecError +from services.studio_image_spec import WangpProcessorSettings, _validate_reference +from services.tools_upscale import TOOL_UPSCALE_METHODS + + +SCHEMA_VERSION = 2 +FINGERPRINT_VERSION = 2 +OPERATION = "tools.upscale" + +_MAX_ID_LENGTH = 240 +_MAX_INTENT_LENGTH = 160 +_MAX_REFERENCE_LENGTH = 8192 +_SEED_MIN = -(2**63) +_SEED_MAX = 2**63 - 1 + +TOOLS_UPSCALE_DEFAULTS: dict[str, Any] = { + "seed": -1, + "wangp_processor_settings": {}, +} + +SUPPORTED_SOURCE_KINDS = ("image", "video") +SUPPORTED_METHODS = tuple(sorted(TOOL_UPSCALE_METHODS)) + +# These processors are explicitly temporal or require a video in the existing +# Tools implementation. The definitive availability/configuration decision +# remains the WangGP processor validator in the preparation boundary. +_VIDEO_ONLY_METHODS = frozenset( + method + for method in TOOL_UPSCALE_METHODS + if method == "h3facerefine" or method.startswith(("rife", "dlssg")) +) + + +class ToolsUpscaleSpecError(ImageGenerationSpecError): + """Validation error for the closed Tools upscale envelope.""" + + +class _ClosedModel(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + +_Workspace = Annotated[ + StrictStr, + StringConstraints( + min_length=1, + max_length=_MAX_ID_LENGTH, + pattern=r"^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$", + ), +] +_WorkspaceCollectionId = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=200), +] +_IntentId = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=_MAX_INTENT_LENGTH), +] +_Source = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=_MAX_REFERENCE_LENGTH), +] +_Method = Annotated[ + StrictStr, + StringConstraints(min_length=1, max_length=80), +] +_Seed = Annotated[StrictInt, Field(ge=_SEED_MIN, le=_SEED_MAX)] + + +class ToolsUpscaleParams(_ClosedModel): + """Typed native input for one image or video upscale.""" + + source: _Source + source_kind: Literal["image", "video"] + method: _Method + seed: _Seed = -1 + wangp_processor_settings: WangpProcessorSettings | None = None + + @field_validator("source") + @classmethod + def _canonical_source(cls, value: str) -> str: + # The shared image reference validator accepts exact asset IDs and the + # same local API URL families used by Studio resources. It rejects + # absolute host paths, remote URLs, fragments, traversal and duplicate + # source-workspace query values without changing caller spelling. + try: + return _validate_reference(value) + except ValueError as error: + raise ValueError(str(error)) from error + + @field_validator("method") + @classmethod + def _known_method(cls, value: str) -> str: + if value not in TOOL_UPSCALE_METHODS: + raise ValueError("method must be one of the installed Tools upscale methods") + return value + + @model_validator(mode="after") + def _check_media_method(self): + if self.source_kind == "image" and self.method in _VIDEO_ONLY_METHODS: + raise ValueError("the selected processor requires a video source") + return self + + +class ToolsUpscaleInput(_ClosedModel): + workspace: _Workspace + # Collection identity is transport provenance. It is retained in the + # receipt/fingerprint but is not forwarded as a native processor option. + workspace_collection_id: _WorkspaceCollectionId | None = None + params: ToolsUpscaleParams + + @model_validator(mode="after") + def _check_collection_id(self): + if self.workspace_collection_id is not None and not self.workspace_collection_id.strip(): + raise ValueError("workspace_collection_id must contain a non-blank value") + return self + + @field_validator("workspace") + @classmethod + def _nonblank_workspace(cls, value: str) -> str: + if not value.strip(): + raise ValueError("input.workspace must be a non-blank output workspace") + return value + + +class _ToolsUpscaleEnvelope(_ClosedModel): + version: Literal[SCHEMA_VERSION] + operation: Literal[OPERATION] + intent_id: _IntentId + input: ToolsUpscaleInput + + +def _validation_error(exc: ValidationError) -> ToolsUpscaleSpecError: + details: list[dict[str, Any]] = [] + messages: list[str] = [] + for error in exc.errors(include_input=False): + location = ".".join(str(part) for part in error.get("loc", ())) or "command" + message = str(error.get("msg") or "Invalid value") + details.append({"loc": location, "message": message, "type": error.get("type")}) + messages.append(f"{location}: {message}") + return ToolsUpscaleSpecError( + "; ".join(messages) or "Invalid Tools upscale command", details=details + ) + + +def _canonical_content(effective: dict[str, Any]) -> dict[str, Any]: + return { + "version": SCHEMA_VERSION, + "operation": OPERATION, + "input": deepcopy(effective["input"]), + } + + +def _fingerprint(content: dict[str, Any]) -> str: + encoded = json.dumps( + content, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _asset_id_from_reference(value: str) -> str | None: + """Return the exact asset ID represented by a source reference.""" + if value.startswith(("asset_", "asset:", "asset-")): + return value + parsed = urlsplit(value) + if parsed.path.startswith("/api/v1/assets/") and not parsed.query: + return unquote(parsed.path[len("/api/v1/assets/"):]) + return None + + +def freeze_tools_upscale_spec(command: Any) -> dict[str, Any]: + """Validate and detach one Tools upscale command without side effects.""" + if type(command) is not dict: + raise ToolsUpscaleSpecError("Tools upscale command must be an object") + try: + envelope = _ToolsUpscaleEnvelope.model_validate(command) + except ValidationError as exc: + raise _validation_error(exc) from exc + + if not envelope.intent_id.strip(): + raise ToolsUpscaleSpecError("intent_id must contain a non-blank value") + + original = deepcopy(command) + explicit_params = envelope.input.params.model_dump(mode="json", exclude_unset=True) + effective_params = deepcopy(explicit_params) + for key, value in TOOLS_UPSCALE_DEFAULTS.items(): + effective_params.setdefault(key, deepcopy(value)) + effective = { + "version": SCHEMA_VERSION, + "operation": OPERATION, + "intent_id": envelope.intent_id, + "input": { + "workspace": envelope.input.workspace, + "params": effective_params, + }, + } + # Preserve omission versus an explicit null, matching the image and + # speech command contracts. The collection id participates in the + # content fingerprint because it identifies the intended collection. + explicit_input = envelope.input.model_dump(mode="json", exclude_unset=True) + if "workspace_collection_id" in explicit_input: + effective["input"]["workspace_collection_id"] = explicit_input["workspace_collection_id"] + return { + "original": original, + "effective": effective, + "fingerprint_version": FINGERPRINT_VERSION, + "fingerprint": _fingerprint(_canonical_content(effective)), + } + + +def tools_upscale_schema() -> dict[str, Any]: + """Return the executable v2 schema used by HTTP, MCP and discovery.""" + input_schema = ToolsUpscaleInput.model_json_schema() + return { + "version": SCHEMA_VERSION, + "operation": OPERATION, + "intent_id": { + "type": "string", + "minLength": 1, + "maxLength": _MAX_INTENT_LENGTH, + }, + "input": input_schema, + "supported_input_fields": [ + "workspace", + "workspace_collection_id", + "source", + "source_kind", + "method", + "seed", + "wangp_processor_settings", + ], + "source_kinds": list(SUPPORTED_SOURCE_KINDS), + "methods": list(SUPPORTED_METHODS), + "effects": deepcopy(TOOLS_UPSCALE_DEFAULTS), + "excluded": [ + "actor", + "client", + "permission", + "provenance", + "workspace inside input.params", + "filesystem paths", + "remote URLs", + "free-form processor settings", + "generation.image", + "generation.speech", + ], + } + + +# Compatibility names used by the other shared-command slices. They all +# point at this one contract and do not create a second schema version. +freeze_tools_upscale_command = freeze_tools_upscale_spec +tools_upscale_schema_v2 = tools_upscale_schema +ToolsUpscaleGenerationInput = ToolsUpscaleInput +ToolsUpscaleGenerationParams = ToolsUpscaleParams + + +__all__ = [ + "FINGERPRINT_VERSION", + "OPERATION", + "SCHEMA_VERSION", + "SUPPORTED_METHODS", + "SUPPORTED_SOURCE_KINDS", + "TOOLS_UPSCALE_DEFAULTS", + "ToolsUpscaleGenerationInput", + "ToolsUpscaleGenerationParams", + "ToolsUpscaleInput", + "ToolsUpscaleParams", + "ToolsUpscaleSpecError", + "freeze_tools_upscale_command", + "freeze_tools_upscale_spec", + "tools_upscale_schema", + "tools_upscale_schema_v2", +] diff --git a/docs/development/TOOLS_COMMANDS.md b/docs/development/TOOLS_COMMANDS.md new file mode 100644 index 000000000..410fa510a --- /dev/null +++ b/docs/development/TOOLS_COMMANDS.md @@ -0,0 +1,64 @@ +# Tools upscale commands + +The first shared Tools operation is `tools.upscale`, version 2. It accepts the +same envelope used by the other shared commands: + +```json +{ + "version": 2, + "operation": "tools.upscale", + "intent_id": "intent-123", + "input": { + "workspace": "my-workspace", + "workspace_collection_id": "collection-123", + "params": { + "source": "/api/v1/file/poster.png?workspace=source", + "source_kind": "image", + "method": "lanczos2", + "seed": -1, + "wangp_processor_settings": {} + } + } +} +``` + +`workspace` is the output workspace. An optional `workspace_collection_id` +identifies the logical collection associated with the command; it is retained +in the receipt and fingerprint and is not a native processor setting. `source` is either an exact asset ID +(`asset_...`) or one canonical local API reference: an upload URL, a +workspace-qualified file URL, or an exact asset URL. Absolute host paths, +remote URLs, traversal, fragments and ambiguous source locations are rejected. +The source kind must be `image` or `video`, and `method` is required rather +than selected by a default. + +The current method names come from the native Tools worker and are published +by `tools_upscale_schema()`. The preparation layer calls the existing WangGP +processor selection and settings validators. `WangpProcessorSettings` is the +only typed settings model; processor prompts retain their literal spelling. +Processor reference images, when declared by the selected processor, are +resolved against their explicit source workspace and recorded with size and +SHA-256 identity after a Pillow verification. Image sources receive the same +verification. Video sources receive the existing `video_editor.probe_media` +probe and must report positive dimensions and duration. + +The freeze result keeps `original` detached and value-preserving. `effective` +adds only `seed=-1` and an empty processor-settings object when omitted. Its +fingerprint covers operation, output workspace and effective parameters while +excluding `intent_id` and caller metadata. The optional collection identity is +part of that fingerprint. Resource identities are attached by the shared +submission service after preparation; the adapter does not create a queue or +write a journal. + +The adapter transfers the prepared request to the existing +`/api/v1/tools/upscale` native endpoint through an in-process callback. The +endpoint remains the authority for its established confined source resolver +and the existing `tools_upscale.py` worker remains the execution authority. +Real mode selects `_run_tool_upscale`; simulation mode selects the existing +`_run_generation` harness through the shared operation adapter. No model or +processor is downloaded by this command contract. + +This slice covers upscale only. Revoice, recast, background removal, music, +audio and model3d require their own explicit contracts and are not advertised +by this catalog entry. Resource preparation proves identity and media +inspectability; final output decoding and processor quality remain execution +QA responsibilities. diff --git a/tests/test_tools_upscale_preparation.py b/tests/test_tools_upscale_preparation.py new file mode 100644 index 000000000..8023d3d11 --- /dev/null +++ b/tests/test_tools_upscale_preparation.py @@ -0,0 +1,282 @@ +"""Provider-free source and processor preparation checks for Tools upscale.""" + +from copy import deepcopy + +import pytest +from fastapi import HTTPException +from PIL import Image + +from services.tools_upscale_commands import ( + _canonical_request_source, + _request_ready_params, + _resolve_source, +) +from services.tools_upscale_preparation import prepare_tools_upscale + + +class FakeResources: + def __init__(self, roots, references): + self.roots = roots + self.references = references + + def workspace_dir(self, name): + return str(self.roots[name]) + + def uploads_dir(self): + return str(self.roots["__uploads__"]) + + def _media(self, value): + path, workspace = self.references[value] + return str(path), workspace + + +@pytest.fixture +def prepared_fixture(tmp_path): + roots = {name: tmp_path / name for name in ("source", "destination", "__uploads__")} + for root in roots.values(): + root.mkdir() + source = roots["source"] / "poster.png" + reference = roots["source"] / "face.png" + Image.new("RGB", (19, 13), "navy").save(source) + Image.new("RGB", (7, 11), "orange").save(reference) + source_url = "/api/v1/file/poster.png?workspace=source" + reference_url = "/api/v1/file/face.png?workspace=source" + resources = FakeResources( + roots, + {source_url: (source, "source"), reference_url: (reference, "source")}, + ) + + def resolver(_params, **_kwargs): + return str(source), source.name, "source", "image", "asset-poster", "destination", str(roots["destination"]) + + def capabilities(): + return [{"value": "lanczos2", "kind": "spatial", "media": ("image",), "enabled": True}] + + def validate(spatial, temporal, image): + assert (spatial, temporal, image) == ("lanczos2", "", True) + return "" + + def settings(method, values): + assert method == "lanczos2" + return dict(values) + + return { + "roots": roots, + "source": source, + "reference": reference, + "source_url": source_url, + "reference_url": reference_url, + "resources": resources, + "resolver": resolver, + "capabilities": capabilities, + "validate": validate, + "settings": settings, + } + + +def _params(fixture, **extra): + params = { + "workspace": "destination", + "source": fixture["source_url"], + "source_kind": "image", + "method": "lanczos2", + "seed": -1, + "wangp_processor_settings": {}, + } + params.update(extra) + return params + + +def _prepare(fixture, params, **extra): + kwargs = { + "resources": fixture["resources"], + "resolve_source": fixture["resolver"], + "processor_capabilities": fixture["capabilities"], + "validate_processors": fixture["validate"], + "processor_settings": fixture["settings"], + } + kwargs.update(extra) + return prepare_tools_upscale(params, **kwargs) + + +def test_image_source_is_confined_inspected_and_snapshotted(prepared_fixture): + params = _params(prepared_fixture) + before = deepcopy(params) + native, resources = _prepare(prepared_fixture, params) + + assert params == before + assert native["source_path"] == str(prepared_fixture["source"]) + assert native["source_workspace"] == "source" + assert native["source_asset_id"] == "asset-poster" + assert native["source_kind"] == "image" + assert native["generation_mode"] == "image" + assert resources[0]["role"] == "source" + assert resources[0]["workspace"] == "source" + assert resources[0]["media"]["width"] == 19 + assert resources[0]["sha256"] + assert resources[0]["size_bytes"] == prepared_fixture["source"].stat().st_size + + +def test_processor_reference_is_resolved_before_validator_and_keeps_order(prepared_fixture): + seen = {} + + def settings(method, values): + seen["values"] = deepcopy(values) + return {"spatial_upsampler_reference_images": values["spatial_upsampler_reference_images"]} + + # A small valid MP4 is unnecessary here: the injected probe is the + # provider-free media boundary, and the source bytes are still hashed. + clip = prepared_fixture["roots"]["source"] / "poster.mp4" + clip.write_bytes(b"fixture video") + native, resources = _prepare( + prepared_fixture, + _params( + prepared_fixture, + method="h3facerefine", + source_kind="video", + source="/api/v1/file/poster.mp4?workspace=source", + wangp_processor_settings={ + "spatial_upsampler_reference_images": [prepared_fixture["reference_url"]], + }, + ), + processor_capabilities=lambda: [{ + "value": "h3facerefine", "kind": "spatial", "media": ("video",), "enabled": True, + }], + validate_processors=lambda spatial, temporal, image: "", + processor_settings=settings, + processor_parameters=lambda _method: [ + {"name": "spatial_upsampler_reference_images", "type": "array"}, + ], + resolve_source=lambda _params, **_kwargs: ( + str(clip), clip.name, "source", "video", "asset-clip", "destination", + str(prepared_fixture["roots"]["destination"]), + ), + probe_video=lambda _path: {"duration": 1.5, "width": 96, "height": 54, "fps": 24}, + ) + + assert native["source_kind"] == "video" + assert seen["values"]["spatial_upsampler_reference_images"] == [ + str(prepared_fixture["reference"]) + ] + assert native["wangp_processor_settings"]["spatial_upsampler_reference_images"] == [ + str(prepared_fixture["reference"]) + ] + assert [item["role"] for item in resources] == ["source", "processor_reference"] + assert resources[1]["index"] == 0 + assert resources[1]["workspace"] == "source" + + +def test_video_source_requires_a_positive_probe(prepared_fixture): + clip = prepared_fixture["roots"]["source"] / "poster.mp4" + clip.write_bytes(b"fixture video") + + def resolver(_params, **_kwargs): + return str(clip), clip.name, "source", "video", "asset-clip", "destination", None + + with pytest.raises(HTTPException, match="duration"): + _prepare( + prepared_fixture, + _params( + prepared_fixture, + source="/api/v1/file/poster.mp4?workspace=source", + source_kind="video", + ), + resolve_source=resolver, + processor_capabilities=lambda: [], + validate_processors=lambda *_args: "", + processor_settings=lambda _method, values: dict(values), + probe_video=lambda _path: {"duration": 0, "width": 96, "height": 54}, + ) + + +def test_invalid_image_is_rejected_before_native_snapshot(prepared_fixture): + invalid = prepared_fixture["roots"]["source"] / "invalid.png" + invalid.write_bytes(b"not an image") + + with pytest.raises(HTTPException, match="could not be decoded"): + _prepare( + prepared_fixture, + _params(prepared_fixture, source="/api/v1/file/invalid.png?workspace=source"), + resolve_source=lambda _params, **_kwargs: ( + str(invalid), invalid.name, "source", "image", "asset-invalid", "destination", None, + ), + ) + + +def test_disabled_or_wrong_media_processor_is_rejected(prepared_fixture): + with pytest.raises(HTTPException, match="disabled"): + _prepare( + prepared_fixture, + _params(prepared_fixture), + processor_capabilities=lambda: [{ + "value": "lanczos2", "kind": "spatial", "media": ("image",), "enabled": False, + }], + ) + + +def test_processor_unknown_setting_is_rejected_without_mutating_input(prepared_fixture): + params = _params(prepared_fixture, wangp_processor_settings={"spatial_upsampler_prompt": "literal"}) + before = deepcopy(params) + with pytest.raises(HTTPException, match="not declared"): + _prepare(prepared_fixture, params, processor_settings=lambda _method, _values: {}) + assert params == before + + +def test_factory_source_adapter_accepts_prepared_asset_id_without_losing_it(): + seen = [] + + def resolver(body, **kwargs): + seen.append((body, kwargs)) + return "resolved" + + resolve = _resolve_source({"_resolve_tool_source": resolver}) + result = resolve({ + "source_kind": "image", + "workspace": "destination", + "asset_id": "asset-poster", + }, expected_kinds=("image",)) + + assert result == "resolved" + assert seen == [( + { + "source_kind": "image", + "workspace": "destination", + "asset_id": "asset-poster", + }, + {"expected_kinds": ("image",)}, + )] + + +def test_native_projection_keeps_managed_asset_id_with_canonical_source(tmp_path): + source_root = tmp_path / "source" + output_root = tmp_path / "destination" + uploads_root = tmp_path / "uploads" + source_root.mkdir() + output_root.mkdir() + uploads_root.mkdir() + source_path = source_root / "poster.png" + source_path.write_bytes(b"already inspected") + params = { + "source": "asset-poster", + "source_kind": "image", + "workspace": "destination", + "source_path": str(source_path), + "source_workspace": "source", + "source_asset_id": "asset-poster", + } + + ready = _request_ready_params(params) + canonical = _canonical_request_source( + params, + { + "_workspace_dir": lambda workspace: str( + {"source": source_root, "destination": output_root}[workspace] + ), + "_uploads_dir": lambda: str(uploads_root), + }, + ) + ready.pop("source_path", None) + ready["source"] = canonical + + assert canonical == "/api/v1/file/poster.png?workspace=source" + assert ready["asset_id"] == "asset-poster" diff --git a/tests/test_tools_upscale_spec.py b/tests/test_tools_upscale_spec.py new file mode 100644 index 000000000..521bfe698 --- /dev/null +++ b/tests/test_tools_upscale_spec.py @@ -0,0 +1,149 @@ +"""Provider-free regressions for the typed Tools upscale envelope.""" + +from copy import deepcopy + +import pytest + +from services.tools_upscale_spec import ( + ToolsUpscaleSpecError, + freeze_tools_upscale_spec, + tools_upscale_schema, +) + + +def command(*, intent="intent-a", source="/api/v1/file/poster.png?workspace=source", + source_kind="image", method="lanczos2", **params): + native = {"source": source, "source_kind": source_kind, "method": method, **params} + return { + "version": 2, + "operation": "tools.upscale", + "intent_id": intent, + "input": {"workspace": "destination", "params": native}, + } + + +def test_freeze_detaches_original_and_adds_only_deterministic_defaults(): + submitted = command() + before = deepcopy(submitted) + frozen = freeze_tools_upscale_spec(submitted) + + assert frozen["original"] == before + assert frozen["original"] is not submitted + assert frozen["effective"]["input"]["params"]["seed"] == -1 + assert frozen["effective"]["input"]["params"]["wangp_processor_settings"] == {} + assert frozen["original"]["input"]["params"] == before["input"]["params"] + submitted["input"]["params"]["source"] = "/api/v1/uploads/other.png" + assert frozen["original"] == before + + +def test_fingerprint_excludes_intent_but_includes_source_kind_and_method(): + first = freeze_tools_upscale_spec(command(intent="one")) + second = freeze_tools_upscale_spec(command(intent="two")) + different_method = freeze_tools_upscale_spec(command(intent="three", method="lanczos1.5")) + different_kind = freeze_tools_upscale_spec( + command(intent="four", source="/api/v1/file/shot.mp4?workspace=source", source_kind="video") + ) + + assert first["fingerprint"] == second["fingerprint"] + assert first["fingerprint"] != different_method["fingerprint"] + assert first["fingerprint"] != different_kind["fingerprint"] + + +def test_collection_identity_is_preserved_and_fingerprinted(): + first_command = command() + first_command["input"]["workspace_collection_id"] = "collection-one" + second_command = command(intent="different-intent") + second_command["input"]["workspace_collection_id"] = "collection-one" + different_collection = command(intent="another-intent") + different_collection["input"]["workspace_collection_id"] = "collection-two" + + first = freeze_tools_upscale_spec(first_command) + second = freeze_tools_upscale_spec(second_command) + other = freeze_tools_upscale_spec(different_collection) + + assert first["original"]["input"]["workspace_collection_id"] == "collection-one" + assert first["effective"]["input"]["workspace_collection_id"] == "collection-one" + assert "workspace_collection_id" not in first["effective"]["input"]["params"] + assert first["fingerprint"] == second["fingerprint"] + assert first["fingerprint"] != other["fingerprint"] + first_command["input"]["workspace_collection_id"] = "changed-after-freeze" + assert first["effective"]["input"]["workspace_collection_id"] == "collection-one" + + +def test_collection_identity_distinguishes_omission_from_explicit_null(): + omitted = freeze_tools_upscale_spec(command()) + explicit_null_command = command(intent="explicit-null") + explicit_null_command["input"]["workspace_collection_id"] = None + explicit_null = freeze_tools_upscale_spec(explicit_null_command) + + assert "workspace_collection_id" not in omitted["effective"]["input"] + assert explicit_null["effective"]["input"]["workspace_collection_id"] is None + assert omitted["fingerprint"] != explicit_null["fingerprint"] + + +@pytest.mark.parametrize("value", ["", " ", True, 12, [], {}]) +def test_collection_identity_is_strict_and_nonblank(value): + values = command() + values["input"]["workspace_collection_id"] = value + with pytest.raises(ToolsUpscaleSpecError, match="workspace_collection_id"): + freeze_tools_upscale_spec(values) + + +@pytest.mark.parametrize( + "bad", + [ + {"source": "/tmp/poster.png"}, + {"source": "https://remote.invalid/poster.png?workspace=source"}, + {"source": "/api/v1/file/../poster.png?workspace=source"}, + {"source": "/api/v1/file/poster.png?workspace=source&workspace=source"}, + {"source": "/api/v1/file/poster.png"}, + {"source": "/api/v1/uploads/poster.png?workspace=source"}, + {"method": "unknown"}, + {"seed": True}, + {"seed": 1.0}, + {"wangp_processor_settings": {"unknown": 1}}, + {"wangp_processor_settings": {"spatial_upsampler_strength": float("nan")}}, + ], +) +def test_freeze_rejects_noncanonical_or_untyped_values_before_effects(bad): + values = command() + values["input"]["params"].update(bad) + with pytest.raises(ToolsUpscaleSpecError): + freeze_tools_upscale_spec(values) + + +def test_video_only_method_is_rejected_for_image_source(): + with pytest.raises(ToolsUpscaleSpecError, match="video source"): + freeze_tools_upscale_spec(command(method="rife2")) + + +def test_processor_prompt_and_references_are_value_preserving(): + prompt = "Face line 1\n Face line 2 " + source = "/api/v1/file/ref.png?workspace=source" + frozen = freeze_tools_upscale_spec(command( + method="h3facerefine", + source_kind="video", + source="/api/v1/file/shot.mp4?workspace=source", + wangp_processor_settings={ + "spatial_upsampler_prompt": prompt, + "spatial_upsampler_reference_images": [source], + }, + )) + + settings = frozen["effective"]["input"]["params"]["wangp_processor_settings"] + assert settings["spatial_upsampler_prompt"] == prompt + assert settings["spatial_upsampler_reference_images"] == [source] + + +def test_schema_publishes_only_the_closed_tools_surface(): + schema = tools_upscale_schema() + params = schema["input"]["$defs"]["ToolsUpscaleParams"] + assert schema["version"] == 2 + assert schema["operation"] == "tools.upscale" + assert params["additionalProperties"] is False + assert set(schema["supported_input_fields"]) == { + "workspace", "workspace_collection_id", "source", "source_kind", "method", "seed", + "wangp_processor_settings", + } + assert "actor" in schema["excluded"] + assert "filesystem paths" in schema["excluded"] From af2ce041afeba08143b698c488e83f5542d1af6b Mon Sep 17 00:00:00 2001 From: IAnMove <216241348+IAnMove@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:11:39 +0200 Subject: [PATCH 04/59] feat(tools): admit upscale through canonical commands and native worker --- app/_launch_runtime.py | 18 +++ app/services/image_generation_commands.py | 93 ++++++++++-- app/services/image_generation_runtime.py | 7 +- app/services/native_generation_operation.py | 3 + .../fixtures/architecture_wire_inventory.json | 6 + tests/test_h3_preplan_job_contract.py | 1 + tests/test_tools_command_runtime.py | 139 ++++++++++++++++++ 7 files changed, 254 insertions(+), 13 deletions(-) create mode 100644 tests/test_tools_command_runtime.py diff --git a/app/_launch_runtime.py b/app/_launch_runtime.py index 26f874029..da761a040 100644 --- a/app/_launch_runtime.py +++ b/app/_launch_runtime.py @@ -10672,6 +10672,13 @@ def _run_generation_with_preparation(job_id: str) -> bool: job = _jobs.get(job_id) if not isinstance(job, dict): return False + try: + native_worker = _image_generation_commands.native_worker(job) + except HTTPException as error: + finish_job(job, "failed", error=str(error.detail), message="Command recovery could not be verified") + return False + if native_worker is not None: + return bool(native_worker(job_id)) params = job.get("params") if isinstance(job.get("params"), dict) else {} pending = params.get("_h3_window_plan_pending") if not isinstance(pending, dict): @@ -23436,6 +23443,17 @@ async def tools_upscale(request: Request): "workspace": workspace, "out_dir": output_dir, "provenance": provenance, } + admit_command = getattr(request, "admit_generation_command", None) + if callable(admit_command): + # Only the typed in-process command adapter can transfer admission. + # Preserve the existing tool's resolved inputs and native worker; + # canonical task/receipt persistence now owns its queue lifecycle. + job["params"].pop("_non_durable_tool", None) + provenance["capability"] = "tools.upscale" + command_collection = (body.get("provenance") or {}).get("workspace_id") + if command_collection is not None: + provenance["workspace_id"] = command_collection + return admit_command(job["params"], workspace, provenance) _register_manual_generation_job(job) worker = _run_generation if execution_mode.policy().simulated else _run_tool_upscale threading.Thread(target=worker, args=(job_id,), daemon=False).start() diff --git a/app/services/image_generation_commands.py b/app/services/image_generation_commands.py index 23c79fba3..87dabbaa4 100644 --- a/app/services/image_generation_commands.py +++ b/app/services/image_generation_commands.py @@ -99,7 +99,9 @@ def _admit(self, frozen, body, workspace, provenance): registry = self._registry(workspace) provenance = deepcopy(provenance) provenance["command"]["command_id"] = frozen["original"]["intent_id"] - native_params = {**deepcopy(self.runtime_defaults()), **deepcopy(body)} + adapter = self.operations.get(frozen["original"]["operation"]) + defaults = self.runtime_defaults() if adapter is None or adapter.use_generation_defaults else {} + native_params = {**deepcopy(defaults), **deepcopy(body)} job = self.make_job(native_params, workspace, reserve_generation=False, publish_task=False, provenance=provenance) effective = deepcopy(frozen["effective"]) effective["runtime"] = {"params": deepcopy(job["params"]), "workspace": workspace, @@ -154,7 +156,8 @@ async def submit(self, command, *, trusted_tool=None, submission_context=None): # The native facade performs its ordinary validation first and then # transfers admission to the same canonical task/worker adapter. request.admit_generation_command = lambda body, workspace, provenance: self._admit(frozen, body, workspace, provenance) - return await self.prepare(request) + prepare_request = adapter.prepare_request if adapter and adapter.prepare_request else self.prepare + return await prepare_request(request) except ImageGenerationSpecError as error: raise command_error(422, "invalid_command", str(error)) from error except TaskCommandConflict as error: @@ -189,8 +192,33 @@ def receipt(self, workspace, intent_id): except (OSError, sqlite3.Error) as error: raise command_error(503, "storage_unavailable", "Command storage is unavailable") from error + def native_worker(self, job): + """Select a tool worker only for its real durable admission. + + Public generation JSON and legacy provenance can never select a tool + by themselves. Both normal dispatch and queue recovery check the + existing canonical receipt before entering the registered worker. + """ + provenance = job.get("provenance") + if not isinstance(provenance, dict) or not isinstance(provenance.get("capability"), str): + return None + operation = provenance["capability"] + adapter = self.operations.get(operation) + if adapter is None or adapter.worker is None: + return None + linked = self._recovery_task(job) + if not linked or linked[1] is None: + raise command_error(503, "recovery_mismatch", "Tool worker requires its canonical task") + return adapter.worker + def restore_recovery(self, workspaces): """Rebuild only the existing recovery projection; never start inference.""" + try: + self._restore_recovery(workspaces) + except (OSError, sqlite3.Error) as error: + raise command_error(503, "storage_unavailable", "Recovery storage is unavailable; no queue records were discarded") from error + + def _restore_recovery(self, workspaces): active = set(self.active_job_ids()) for workspace in workspaces: registry = self._registry(workspace) @@ -208,21 +236,62 @@ def restore_recovery(self, workspaces): self.persist_recovery({"id": task["backend_job_id"], "status": "interrupted", "created_at": task["created_at"], **deepcopy(runtime)}) - def _recovery_task(self, record): - provenance = record.get("provenance") or {} - if provenance.get("capability") not in {"generation.image", *self.operations}: + def _recovery_identity(self, record): + if not isinstance(record, dict): + return False + provenance = record.get("provenance") + if provenance is None: return None - registry = self._registry(record["workspace"]) - intent_id = provenance.get("command", {}).get("command_id") - entry = registry.command_admission(intent_id) - if entry is None or entry["receipt"]["result"]["job_id"] != record["id"]: - raise command_error(503, "recovery_mismatch", "Recovery does not match a durable generation admission") - return registry, registry.get(entry["task_id"]) + if not isinstance(provenance, dict): + return False + capability = provenance.get("capability") + if capability is not None and not isinstance(capability, str): + return False + if capability not in {"generation.image", *self.operations}: + return None + command = provenance.get("command") + if not isinstance(command, dict): + return False + intent_id = command.get("command_id") + if not isinstance(intent_id, str) or not 1 <= len(intent_id) <= 160 or not intent_id.strip(): + return False + return intent_id + + def _recovery_task(self, record): + """Link one leftover to its admission, or withhold it. + + Unregistered operations return None for legacy native recovery. + A linked command returns ``(registry, task)``. A registered row that + cannot be matched (missing admission, invalid queue metadata or job-id + drift) returns False so this one row is skipped. Storage failures, + including corrupt canonical admissions, remain errors: discard must + not delete recovery records while their tasks cannot be verified. + """ + intent_id = self._recovery_identity(record) + if intent_id is None or intent_id is False: + return intent_id + try: + registry = self._registry(record.get("workspace")) + entry = registry.command_admission(intent_id) + if (entry is None or entry["operation"] != record["provenance"]["capability"] + or entry["receipt"]["result"]["job_id"] != record.get("id")): + return False + return registry, registry.get(entry["task_id"]) + except HTTPException as error: + if error.status_code in {404, 422}: + return False + raise + except (OSError, sqlite3.Error) as error: + raise command_error(503, "storage_unavailable", "Recovery storage is unavailable; no queue records were discarded") from error + except (TypeError, KeyError): + return False def filter_recovery(self, records): retained = [] for record in records: linked = self._recovery_task(record) + if linked is False: + continue if linked is None or (linked[1] and linked[1]["status"] == "interrupted"): retained.append(record) return retained @@ -230,7 +299,7 @@ def filter_recovery(self, records): def discard_recovery(self, records): for record in records: linked = self._recovery_task(record) - if linked is not None and linked[1] and linked[1]["status"] == "interrupted": + if linked and linked[1] and linked[1]["status"] == "interrupted": registry, task = linked registry.update(task["id"], status="cancelled", phase="recovery_discarded", message="Recovery discarded", completed_at=time.time(), recoverable=False) diff --git a/app/services/image_generation_runtime.py b/app/services/image_generation_runtime.py index eff3af7c2..74b9e6af3 100644 --- a/app/services/image_generation_runtime.py +++ b/app/services/image_generation_runtime.py @@ -83,12 +83,17 @@ def prepare(params): return NativeGenerationOperation(freeze=freeze, prepare=prepare, catalog=speech_command_catalog()) + operations = {"generation.speech": speech_operation()} + if callable(runtime.get("tools_upscale")) and callable(runtime.get("_run_tool_upscale")): + from services.tools_upscale_commands import create_tools_upscale_operation + operations["tools.upscale"] = create_tools_upscale_operation(runtime) + service = ImageGenerationCommands( registry=runtime["_task_registry"], prepare=runtime["generate"], preflight=preflight, make_job=runtime["_new_generation_job"], task_fields=runtime["_generation_task_fields"], dispatch=dispatch, persist_recovery=persist, active_job_ids=lambda: runtime["_jobs"].keys(), prepare_studio=prepare_studio, runtime_defaults=lambda: {"mode": "", **runtime["wgp"].primary_settings}, - operations={"generation.speech": speech_operation()}, + operations=operations, ) service.canonicalize_reference = lambda value, media_kind="image": resources(media_kind).canonicalize_legacy(value) return service diff --git a/app/services/native_generation_operation.py b/app/services/native_generation_operation.py index a874f3901..17f78def2 100644 --- a/app/services/native_generation_operation.py +++ b/app/services/native_generation_operation.py @@ -12,3 +12,6 @@ class NativeGenerationOperation: freeze: Callable[[dict], tuple[dict, dict]] prepare: Callable[[dict], tuple[dict, list]] catalog: dict + prepare_request: Callable | None = None + worker: Callable[[str], bool] | None = None + use_generation_defaults: bool = True diff --git a/tests/fixtures/architecture_wire_inventory.json b/tests/fixtures/architecture_wire_inventory.json index 4cb66e674..437eb999a 100644 --- a/tests/fixtures/architecture_wire_inventory.json +++ b/tests/fixtures/architecture_wire_inventory.json @@ -367,6 +367,12 @@ "classification": "fragile_source", "reason": "Reads launch text directly and may need conversion when the referenced domain is extracted." }, + { + "file": "tests/test_tools_command_runtime.py", + "target": "app/_launch_runtime.py", + "classification": "symbol_importable", + "reason": "Extracts selected launch symbols with AST/exec; migrate to direct imports when that domain moves." + }, { "file": "tests/test_tools_upscale_contract.py", "target": "app/_launch_runtime.py", diff --git a/tests/test_h3_preplan_job_contract.py b/tests/test_h3_preplan_job_contract.py index 1e3a38341..547d8fdb6 100644 --- a/tests/test_h3_preplan_job_contract.py +++ b/tests/test_h3_preplan_job_contract.py @@ -323,6 +323,7 @@ def acknowledge_cancel(job: dict, **updates) -> bool: "acknowledge_cancel": acknowledge_cancel, "snapshot_job": lambda job: dict(job), "_run_generation": lambda job_id, **_kwargs: gpu_calls.append(job_id), + "_image_generation_commands": SimpleNamespace(native_worker=lambda _job: None), } _load( "_new_generation_job", diff --git a/tests/test_tools_command_runtime.py b/tests/test_tools_command_runtime.py new file mode 100644 index 000000000..dd59c5935 --- /dev/null +++ b/tests/test_tools_command_runtime.py @@ -0,0 +1,139 @@ +"""Native Tools route admission and recovery; no model or processor execution.""" +import ast +from copy import deepcopy +from pathlib import Path +from types import SimpleNamespace +import threading +import time +import uuid + +from fastapi import FastAPI, HTTPException +from fastapi.testclient import TestClient +import pytest + +from routers.image_generation_commands import create_image_generation_commands_router, image_command_handlers +from routers.tools_upscale_commands import tools_upscale_command_catalog +from services.native_generation_operation import NativeGenerationOperation +from services.tools_upscale import TOOL_UPSCALE_METHODS +from services.tools_upscale_spec import freeze_tools_upscale_spec +from tests.test_image_generation_commands import FakeNative, _command as image_command, _run + + +def command(intent="upscale-intent"): + return {"version": 2, "operation": "tools.upscale", "intent_id": intent, + "input": {"workspace": "tool-destination", "params": { + "source": "/api/v1/file/source.png?workspace=tool-source", + "source_kind": "image", "method": "lanczos2", "seed": 42}}} + + +def native_endpoint(tmp_path, legacy_calls): + source = Path(__file__).resolve().parents[1] / "app" / "_launch_runtime.py" + tree = ast.parse(source.read_text(encoding="utf-8")) + definition = next(node for node in tree.body if isinstance(node, ast.AsyncFunctionDef) and node.name == "tools_upscale") + definition.decorator_list = [] + namespace = { + "Request": object, "HTTPException": HTTPException, "uuid": uuid, "time": time, + "threading": threading, "_TOOL_UPSCALE_METHODS": TOOL_UPSCALE_METHODS, + "_resolve_tool_source": lambda body, **_kwargs: ( + str(tmp_path / "source.png"), "source.png", "tool-source", body["source_kind"], + "asset_source", body["workspace"], str(tmp_path / "destination")), + "_register_manual_generation_job": lambda job: legacy_calls.append(job), + "execution_mode": SimpleNamespace(policy=lambda: SimpleNamespace(simulated=False)), + } + exec(compile(ast.Module(body=[definition], type_ignores=[]), str(source), "exec"), namespace) + return namespace["tools_upscale"] + + +def configured_service(native, tmp_path): + service = native.service() + legacy_calls, worker_calls = [], [] + + def freeze(body): + frozen = freeze_tools_upscale_spec(body) + original_input = frozen["effective"]["input"] + return frozen, {**deepcopy(original_input["params"]), "workspace": original_input["workspace"]} + + service.runtime_defaults = lambda: {"prompt": "unrelated image form", "model_type": "must-not-leak"} + service.operations["tools.upscale"] = NativeGenerationOperation( + freeze=freeze, prepare=lambda params: (params, []), catalog=tools_upscale_command_catalog(), + prepare_request=native_endpoint(tmp_path, legacy_calls), + worker=lambda job_id: worker_calls.append(job_id) or True, use_generation_defaults=False, + ) + return service, legacy_calls, worker_calls + + +def test_tools_http_and_mcp_share_the_native_route_and_exact_receipt(tmp_path): + native = FakeNative(tmp_path) + service, legacy, workers = configured_service(native, tmp_path) + app = FastAPI() + app.include_router(create_image_generation_commands_router(service)) + with TestClient(app) as client: + response = client.post("/api/v1/generation/commands", json=command(), + headers={"X-Hocus-UI-Surface": "wizard"}) + assert response.status_code == 200 + first = response.json() + second = _run(image_command_handlers(service)["tools.upscale"]( + {key: value for key, value in command().items() if key != "operation"})) + assert first["receipt"] == second["receipt"] + assert second["replayed"] is True + assert len(native.dispatch_calls) == 1 + job = native.dispatch_calls[0] + assert job["params"]["model_type"] == "post_processing" + assert "prompt" not in job["params"] + assert "_non_durable_tool" not in job["params"] + assert job["params"]["source_workspace"] == "tool-source" + assert job["provenance"]["capability"] == "tools.upscale" + assert job["provenance"]["actor"] == "wizard" + assert service.native_worker(job)(job["id"]) is True + assert workers == [job["id"]] + assert legacy == [] + assert native.prepare_calls == 0 + another = _run(service.submit(command("another-deliberate-upscale"))) + assert another["receipt"]["taskIds"] != first["receipt"]["taskIds"] + + +def test_tool_recovery_retains_native_snapshot_and_requires_exact_admission(tmp_path): + native = FakeNative(tmp_path) + service, _, _ = configured_service(native, tmp_path) + accepted = _run(service.submit(command())) + restarted = FakeNative(tmp_path, interrupt_stale=True) + recovery, legacy, workers = configured_service(restarted, tmp_path) + recovery.restore_recovery(["tool-destination"]) + assert len(restarted.persist_calls) == 1 + record = restarted.persist_calls[0] + assert recovery.filter_recovery([record]) == [record] + assert record["params"] == native.dispatch_calls[0]["params"] + assert recovery.native_worker(record)(record["id"]) is True + assert workers == [record["id"]] + forged = deepcopy(record) + forged["provenance"]["command"]["command_id"] = "not-admitted" + with pytest.raises(HTTPException) as caught: + recovery.native_worker(forged) + assert caught.value.status_code == 503 + assert workers == [record["id"]] + assert legacy == [] + assert _run(recovery.submit(command()))["receipt"] == accepted["receipt"] + assert restarted.dispatch_calls == [] + + +def test_tool_keeps_collection_attribution_separate_from_physical_output_workspace(tmp_path): + native = FakeNative(tmp_path) + service, _, _ = configured_service(native, tmp_path) + request = command() + request["input"]["workspace_collection_id"] = "collection-curated" + accepted = _run(service.submit(request)) + job = native.dispatch_calls[0] + assert job["workspace"] == "tool-destination" + assert accepted["receipt"]["result"]["workspace"] == "tool-destination" + assert job["provenance"]["workspace_id"] == "collection-curated" + + +def test_image_admission_cannot_be_relabelled_as_a_tool_worker(tmp_path): + native = FakeNative(tmp_path) + service, _, workers = configured_service(native, tmp_path) + _run(service.submit(image_command())) + record = deepcopy(native.dispatch_calls[0]) + record["provenance"]["capability"] = "tools.upscale" + with pytest.raises(HTTPException): + service.native_worker(record) + assert workers == [] From ba974fbd6177fa2a10d11b625a6699a3662f7f31 Mon Sep 17 00:00:00 2001 From: IAnMove <216241348+IAnMove@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:14:40 +0200 Subject: [PATCH 05/59] refactor(tools): separate source projection from source validation --- app/services/tools_upscale_preparation.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/services/tools_upscale_preparation.py b/app/services/tools_upscale_preparation.py index ff47a87f1..f92b9280d 100644 --- a/app/services/tools_upscale_preparation.py +++ b/app/services/tools_upscale_preparation.py @@ -95,7 +95,7 @@ def _fallback_source(params: dict[str, Any], resources: Any): } -def _normalise_source(result: Any, expected_kind: str) -> dict[str, Any]: +def _source_fields(result: Any) -> dict[str, Any]: if isinstance(result, Mapping): values = { "path": result.get("path") or result.get("source_path") or result.get("resolved"), @@ -115,6 +115,11 @@ def _normalise_source(result: Any, expected_kind: str) -> dict[str, Any]: } else: raise ValueError("The Tools source resolver returned an invalid result") + return values + + +def _normalise_source(result: Any, expected_kind: str) -> dict[str, Any]: + values = _source_fields(result) path = values["path"] if not isinstance(path, (str, os.PathLike)) or not os.fspath(path): raise ValueError("The selected Tools source is unavailable") From 9a157fecae9c07cccbf74e645c8bd52f5c93a28a Mon Sep 17 00:00:00 2001 From: IAnMove <216241348+IAnMove@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:19:49 +0200 Subject: [PATCH 06/59] fix(tools): identify durable upscale tasks in Activity --- app/_launch_runtime.py | 2 +- docs/development/TOOLS_COMMANDS.md | 5 +++-- tests/test_tools_command_runtime.py | 35 +++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/app/_launch_runtime.py b/app/_launch_runtime.py index da761a040..1040d9c39 100644 --- a/app/_launch_runtime.py +++ b/app/_launch_runtime.py @@ -36242,7 +36242,7 @@ def _generation_task_fields(job: dict) -> dict: }.get(mode, "Generation job") if str(provenance.get("capability") or "") == "remove_background": task_title = "Tools · Remove background" - elif str(provenance.get("capability") or "") == "upscale": + elif str(provenance.get("capability") or "") in {"upscale", "tools.upscale"}: task_title = "Tools · Upscale" elif str(provenance.get("capability") or "") == "revoice": task_title = "Tools · Revoice" diff --git a/docs/development/TOOLS_COMMANDS.md b/docs/development/TOOLS_COMMANDS.md index 410fa510a..6c9afaa64 100644 --- a/docs/development/TOOLS_COMMANDS.md +++ b/docs/development/TOOLS_COMMANDS.md @@ -24,7 +24,7 @@ same envelope used by the other shared commands: `workspace` is the output workspace. An optional `workspace_collection_id` identifies the logical collection associated with the command; it is retained -in the receipt and fingerprint and is not a native processor setting. `source` is either an exact asset ID +in the durable command snapshots, provenance and fingerprint and is not a native processor setting. `source` is either an exact asset ID (`asset_...`) or one canonical local API reference: an upload URL, a workspace-qualified file URL, or an exact asset URL. Absolute host paths, remote URLs, traversal, fragments and ambiguous source locations are rejected. @@ -45,7 +45,8 @@ The freeze result keeps `original` detached and value-preserving. `effective` adds only `seed=-1` and an empty processor-settings object when omitted. Its fingerprint covers operation, output workspace and effective parameters while excluding `intent_id` and caller metadata. The optional collection identity is -part of that fingerprint. Resource identities are attached by the shared +part of that fingerprint. The public receipt reports the physical output workspace; +collection identity is not a separate public receipt field. Resource identities are attached by the shared submission service after preparation; the adapter does not create a queue or write a journal. diff --git a/tests/test_tools_command_runtime.py b/tests/test_tools_command_runtime.py index dd59c5935..f09bbeb7a 100644 --- a/tests/test_tools_command_runtime.py +++ b/tests/test_tools_command_runtime.py @@ -111,6 +111,13 @@ def test_tool_recovery_retains_native_snapshot_and_requires_exact_admission(tmp_ recovery.native_worker(forged) assert caught.value.status_code == 503 assert workers == [record["id"]] + drifted = deepcopy(record) + drifted["id"] = "different-native-job" + with pytest.raises(HTTPException) as mismatch: + recovery.native_worker(drifted) + assert mismatch.value.status_code == 503 + assert mismatch.value.detail["code"] == "recovery_mismatch" + assert workers == [record["id"]] assert legacy == [] assert _run(recovery.submit(command()))["receipt"] == accepted["receipt"] assert restarted.dispatch_calls == [] @@ -137,3 +144,31 @@ def test_image_admission_cannot_be_relabelled_as_a_tool_worker(tmp_path): with pytest.raises(HTTPException): service.native_worker(record) assert workers == [] + + +@pytest.mark.parametrize("capability", ["upscale", "tools.upscale"]) +def test_native_task_projection_identifies_upscale_for_activity(tmp_path, capability): + source = Path(__file__).resolve().parents[1] / "app" / "_launch_runtime.py" + tree = ast.parse(source.read_text(encoding="utf-8")) + definition = next(node for node in tree.body if isinstance(node, ast.FunctionDef) + and node.name == "_generation_task_fields") + namespace = { + "time": time, + "_public_generation_details": lambda params: params, + "_task_status": lambda status: status, + "_task_timestamp": lambda job, key: job.get(key), + "_canonical_legacy_progress": lambda *_args: 0, + "_is_durable_generation_job": lambda _job: True, + "_local_gpu_lane": SimpleNamespace(key="local_gpu:0"), + } + exec(compile(ast.Module(body=[definition], type_ignores=[]), str(source), "exec"), namespace) + task = namespace["_generation_task_fields"]({ + "id": "upscale-job", "status": "queued", "workspace": "tool-destination", + "params": {"generation_mode": "image", "model_type": "post_processing"}, + "provenance": {"capability": capability, "command": {"command_id": "tool-intent"}}, + }) + assert task["title"] == "Tools · Upscale" + assert task["metadata"]["capability"] == capability + assert task["metadata"]["command_id"] == "tool-intent" + assert task["workspace"] == "tool-destination" + assert task["backend_job_id"] == "upscale-job" From 1868f107066a92720c14c6438cc7216f07ce0679 Mon Sep 17 00:00:00 2001 From: IAnMove <216241348+IAnMove@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:23:41 +0200 Subject: [PATCH 07/59] fix(tools): keep inactive processor defaults out of native requests --- app/services/tools_upscale_preparation.py | 5 ++++- docs/development/TOOLS_COMMANDS.md | 7 ++++--- tests/test_tools_upscale_preparation.py | 13 +++++++++++++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/app/services/tools_upscale_preparation.py b/app/services/tools_upscale_preparation.py index f92b9280d..73b1ec507 100644 --- a/app/services/tools_upscale_preparation.py +++ b/app/services/tools_upscale_preparation.py @@ -48,7 +48,10 @@ def _validated_params(params: Any) -> tuple[dict[str, Any], str]: ) raise command_error(422, "invalid_tools_upscale_input", details) from error working = parsed.model_dump(mode="json") - working["wangp_processor_settings"] = working.get("wangp_processor_settings") or {} + # Only submitted settings belong to this processor. Nested model defaults + # include an inactive H3 reference list which is not a Lanczos parameter. + settings = parsed.wangp_processor_settings + working["wangp_processor_settings"] = settings.model_dump(mode="json", exclude_unset=True) if settings else {} working["workspace"] = workspace return working, workspace diff --git a/docs/development/TOOLS_COMMANDS.md b/docs/development/TOOLS_COMMANDS.md index 6c9afaa64..e09294d31 100644 --- a/docs/development/TOOLS_COMMANDS.md +++ b/docs/development/TOOLS_COMMANDS.md @@ -35,9 +35,10 @@ The current method names come from the native Tools worker and are published by `tools_upscale_schema()`. The preparation layer calls the existing WangGP processor selection and settings validators. `WangpProcessorSettings` is the only typed settings model; processor prompts retain their literal spelling. -Processor reference images, when declared by the selected processor, are -resolved against their explicit source workspace and recorded with size and -SHA-256 identity after a Pillow verification. Image sources receive the same +The preparation adapter resolves processor reference images against their +explicit source workspace and records size and SHA-256 identity after Pillow +verification. The current native scalar-settings validator still rejects H3 +reference-image arrays; that option is not yet end-to-end supported. Image sources receive the same verification. Video sources receive the existing `video_editor.probe_media` probe and must report positive dimensions and duration. diff --git a/tests/test_tools_upscale_preparation.py b/tests/test_tools_upscale_preparation.py index 8023d3d11..2a8e13a4d 100644 --- a/tests/test_tools_upscale_preparation.py +++ b/tests/test_tools_upscale_preparation.py @@ -280,3 +280,16 @@ def test_native_projection_keeps_managed_asset_id_with_canonical_source(tmp_path assert canonical == "/api/v1/file/poster.png?workspace=source" assert ready["asset_id"] == "asset-poster" + + +@pytest.mark.parametrize("settings", [None, {}, {"spatial_upsampler_prompt": None}]) +def test_empty_settings_cross_real_scalar_processor_validator(prepared_fixture, settings): + from shared.wangp1272.processors import validated_settings + + native, _ = _prepare( + prepared_fixture, + _params(prepared_fixture, wangp_processor_settings=settings), + processor_settings=validated_settings, + processor_parameters=lambda _method: [], + ) + assert native["wangp_processor_settings"] == {} From 98a585c11e377d30c8e4e7fe03f9942fe6f712b9 Mon Sep 17 00:00:00 2001 From: IAnMove <216241348+IAnMove@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:36:00 +0200 Subject: [PATCH 08/59] docs: explain Wizard and external MCP command recovery --- docs/development/SHARED_NATIVE_COMMANDS.md | 112 +++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 docs/development/SHARED_NATIVE_COMMANDS.md diff --git a/docs/development/SHARED_NATIVE_COMMANDS.md b/docs/development/SHARED_NATIVE_COMMANDS.md new file mode 100644 index 000000000..0abe93c92 --- /dev/null +++ b/docs/development/SHARED_NATIVE_COMMANDS.md @@ -0,0 +1,112 @@ +# Use shared commands from Wizard or an MCP client + +Wizard operates inside the web application. MCP accepts explicit operations +from an external client while the application server is running. Closing a +browser does not stop an already admitted native generation or upscale job. +Server-driven editorial workflows are a separate migration and are not covered +by this guarantee. + +## Wizard and manual controls + +1. Choose the destination workspace and select installed models or existing + source media. For Tools, select the source, its kind and the upscale method. +2. Ask Wizard for the operation with those exact identifiers, or use the + ordinary Generate/Run controls. +3. The application presents the request in the relevant Studio/Tools panel. + A workspace or form change during preparation prevents that submission. +4. The queued message contains a task/job identity. Follow Activity to its + terminal status and open the resulting asset. Queued does not mean finished. +5. If a submission response is lost, recover the saved request in the panel. + Recovery reuses its intention; it must not silently create a new generation. + +The shared native routes currently cover image, speech and upscale. Other +Studio modes continue through their existing paths until migrated. Speech +model duration controls have model-specific meanings: for example, a 20-second +Kugel setting does not force a short sentence to occupy exactly 20 seconds. + +## MCP connection and discovery + +Set `HOCUS_MCP_TOKEN` in the server's environment before starting the +application. Configure the external client's HTTP endpoint as +`http://SERVER:PORT/api/v1/wangp/mcp` and its Authorization header as +`Bearer YOUR_TOKEN`. Keep the actual token out of saved requests and reports. +The endpoint is disabled when no token is configured. + +The tested protocol is `2025-03-26`, using the official JavaScript MCP SDK's +Streamable HTTP client. Discovery uses `tools/list`; the returned schemas are +the authority for supported versions and parameters. The server does not invoke +Wizard's LLM to interpret an explicit MCP operation. + +Relevant tools include: + +| Tool | Purpose | +| --- | --- | +| `models`, `processors` | Discover exact model IDs and processor availability | +| `assets` | Find source IDs and workspace-qualified media URLs | +| `generation.image` | Submit the shared image specification | +| `generation.speech` | Submit the shared speech specification | +| `tools.upscale` | Submit a typed upscale request for an existing image/video | +| `generation.receipt` | Recover a shared admission and its canonical task | +| `status` | Follow the returned native job ID | + +The previous `generate`, `upscale` and other legacy tools remain available. +Their schemas differ from the typed operations above; do not mix envelopes. + +For example, call `tools.upscale` with arguments shaped as follows, replacing +the example source with an existing resource obtained through `assets`: + +```json +{ + "version": 2, + "intent_id": "one-client-intention", + "input": { + "workspace": "my-outputs", + "params": { + "source": "/api/v1/file/source.png?workspace=my-inputs", + "source_kind": "image", + "method": "lanczos2", + "seed": 42, + "wangp_processor_settings": {} + } + } +} +``` + +MCP carries the operation in the tool name. The equivalent local HTTP request +adds `"operation": "tools.upscale"` and goes to +`POST /api/v1/generation/commands`. Its catalog is available at the same path +with GET. The collection ID, when supplied, is distinct from the physical +output workspace; see [Tools commands](TOOLS_COMMANDS.md). + +## Follow-up, retry and errors + +Save the command and its `intent_id` before sending. An admission response has +`receipt.result.job_id` and `receipt.result.task_id`. Poll `status` with the +job ID. On completion, its output filenames belong to the explicit destination +workspace; source media remain in their source workspace. + +To recover an uncertain response, call `generation.receipt` with: + +```json +{ + "version": 1, + "input": { + "workspace": "my-outputs", + "intent_id": "one-client-intention" + } +} +``` + +Repeat the exact command with the same intention after a transport failure. +An existing admission returns the same receipt and task. Changing the content +under that intention produces a conflict. A deliberately new generation uses +a new intention, even when the prompt is identical. After a server restart, +inspect the saved receipt and queue recovery before explicitly resuming an +interrupted job. + +Validation errors do not establish admission. A storage/dispatch error can +require recovery, so do not replace its intention automatically. Native task +status remains the completion authority. Read [image](IMAGE_COMMANDS.md), +[speech](SPEECH_COMMANDS.md) and [upscale](TOOLS_COMMANDS.md) contracts for +supported inputs and current limits. Hashes record inspected sources; they do +not make external source files immutable throughout queue lifetime. From 0ce49851e3f280cf65e4779b79fb106a3960a48f Mon Sep 17 00:00:00 2001 From: IAnMove <216241348+IAnMove@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:44:49 +0200 Subject: [PATCH 09/59] fix(tools): scope upscale assets by source workspace --- app/services/tools_upscale_commands.py | 31 ++++++++ app/services/tools_upscale_preparation.py | 5 ++ app/services/tools_upscale_spec.py | 29 ++++++- docs/development/TOOLS_COMMANDS.md | 9 ++- tests/test_tools_upscale_preparation.py | 94 +++++++++++++++++++++++ tests/test_tools_upscale_spec.py | 36 ++++++++- 6 files changed, 200 insertions(+), 4 deletions(-) diff --git a/app/services/tools_upscale_commands.py b/app/services/tools_upscale_commands.py index 8dbb63927..cb28fbbbd 100644 --- a/app/services/tools_upscale_commands.py +++ b/app/services/tools_upscale_commands.py @@ -65,6 +65,36 @@ def _resource_adapter(runtime): ) +def _asset_locations(runtime, asset_id): + roots_factory = runtime.get("_tool_asset_roots") + if not callable(roots_factory): + return None + from services.asset_catalog import find_asset + + asset = find_asset(roots_factory(), asset_id) + if asset is None: + return None + locations = asset.get("locations") + if not isinstance(locations, list): + raise command_error(409, "source_location_unavailable", "The source asset has no valid locations") + return locations + + +def _check_asset_scope(runtime, asset_id, source_workspace): + locations = _asset_locations(runtime, asset_id) + if locations is None: + return + if any(not isinstance(location, dict) for location in locations): + raise command_error(409, "source_location_unavailable", "The source asset has invalid locations") + matches = [item for item in locations if item.get("workspace_id") == source_workspace] + if source_workspace is not None and len(matches) != 1: + code = "ambiguous_source" if len(matches) > 1 else "source_workspace_mismatch" + raise command_error(409, code, "The source_workspace does not identify one exact asset location") + if source_workspace is None and len(locations) != 1: + code = "source_location_unavailable" if not locations else "ambiguous_source" + raise command_error(409, code, "Choose a source_workspace for this asset") + + def _resolve_source(runtime): resolver = runtime.get("_resolve_tool_source") or runtime.get("resolve_tool_source") if not callable(resolver): @@ -82,6 +112,7 @@ def resolve(params, **kwargs): if asset_id: body["asset_id"] = asset_id body.pop("source", None) + _check_asset_scope(runtime, asset_id, body.get("source_workspace")) expected_kinds = kwargs.get("expected_kinds") or (body.get("source_kind"),) try: return resolver(body, expected_kinds=expected_kinds) diff --git a/app/services/tools_upscale_preparation.py b/app/services/tools_upscale_preparation.py index 73b1ec507..daf34d6cb 100644 --- a/app/services/tools_upscale_preparation.py +++ b/app/services/tools_upscale_preparation.py @@ -74,6 +74,8 @@ def _call_source_resolver(resolver: Callable, params: dict[str, Any], kind: str) request["asset_id"] = _asset_id_from_reference(source) request.pop("source", None) + if params.get("source_workspace") is not None: + request["source_workspace"] = params["source_workspace"] try: return resolver(request, expected_kinds=(kind,)) except TypeError as first_error: @@ -373,6 +375,9 @@ def prepare_tools_upscale( result = (_call_source_resolver(resolver, working, kind) if callable(resolver) else _fallback_source(working, resources)) source = _normalise_source(result, kind) + requested_workspace = working.get("source_workspace") + if requested_workspace is not None and source["source_workspace"] != requested_workspace: + raise ValueError("The selected source does not match source_workspace") source["url"] = working["source"] path = _check_source_path(source, resources) identities = [_source_resource(source, path, resources, probe_video)] diff --git a/app/services/tools_upscale_spec.py b/app/services/tools_upscale_spec.py index 617eeb2d6..796f7b93d 100644 --- a/app/services/tools_upscale_spec.py +++ b/app/services/tools_upscale_spec.py @@ -17,6 +17,7 @@ "workspace_collection_id": "...", "params": { "source": "asset_... or /api/v1/...", + "source_workspace": "source or __uploads__", "source_kind": "image" or "video", "method": "lanczos2", "seed": -1, @@ -37,7 +38,7 @@ import hashlib import json from typing import Annotated, Any, Literal -from urllib.parse import unquote, urlsplit +from urllib.parse import parse_qs, unquote, urlsplit from pydantic import ( BaseModel, @@ -112,6 +113,14 @@ class _ClosedModel(BaseModel): StrictStr, StringConstraints(min_length=1, max_length=_MAX_REFERENCE_LENGTH), ] +_SourceWorkspace = Annotated[ + StrictStr, + StringConstraints( + min_length=1, + max_length=160, + pattern=r"^(?:__uploads__|default|[A-Za-z0-9][A-Za-z0-9_-]*)$", + ), +] _Method = Annotated[ StrictStr, StringConstraints(min_length=1, max_length=80), @@ -123,6 +132,9 @@ class ToolsUpscaleParams(_ClosedModel): """Typed native input for one image or video upscale.""" source: _Source + # An asset ID can exist in more than one explicit media root. This scope + # selects that exact source location; it is kept in the native snapshot. + source_workspace: _SourceWorkspace | None = None source_kind: Literal["image", "video"] method: _Method seed: _Seed = -1 @@ -147,6 +159,20 @@ def _known_method(cls, value: str) -> str: raise ValueError("method must be one of the installed Tools upscale methods") return value + @model_validator(mode="after") + def _check_source_workspace(self): + parsed = urlsplit(self.source) + expected = None + if parsed.path.startswith("/api/v1/uploads/"): + expected = "__uploads__" + elif parsed.path.startswith("/api/v1/file/"): + values = parse_qs(parsed.query, keep_blank_values=True).get("workspace", []) + if len(values) == 1: + expected = values[0] + if expected and self.source_workspace not in (None, expected): + raise ValueError("source_workspace must match the source URL workspace") + return self + @model_validator(mode="after") def _check_media_method(self): if self.source_kind == "image" and self.method in _VIDEO_ONLY_METHODS: @@ -280,6 +306,7 @@ def tools_upscale_schema() -> dict[str, Any]: "workspace", "workspace_collection_id", "source", + "source_workspace", "source_kind", "method", "seed", diff --git a/docs/development/TOOLS_COMMANDS.md b/docs/development/TOOLS_COMMANDS.md index e09294d31..580c33fe9 100644 --- a/docs/development/TOOLS_COMMANDS.md +++ b/docs/development/TOOLS_COMMANDS.md @@ -13,6 +13,7 @@ same envelope used by the other shared commands: "workspace_collection_id": "collection-123", "params": { "source": "/api/v1/file/poster.png?workspace=source", + "source_workspace": "source", "source_kind": "image", "method": "lanczos2", "seed": -1, @@ -26,8 +27,12 @@ same envelope used by the other shared commands: identifies the logical collection associated with the command; it is retained in the durable command snapshots, provenance and fingerprint and is not a native processor setting. `source` is either an exact asset ID (`asset_...`) or one canonical local API reference: an upload URL, a -workspace-qualified file URL, or an exact asset URL. Absolute host paths, -remote URLs, traversal, fragments and ambiguous source locations are rejected. +workspace-qualified file URL, or an exact asset URL. `source_workspace` is +optional; when supplied for a source URL it must match that URL's workspace. +Use `__uploads__` for an upload URL. For an asset ID with several catalog locations, omit the scope +only when there is exactly one location, otherwise select one exact +`source_workspace`. Absolute host paths, remote URLs, traversal, fragments, +missing scopes and ambiguous source locations are rejected. The source kind must be `image` or `video`, and `method` is required rather than selected by a default. diff --git a/tests/test_tools_upscale_preparation.py b/tests/test_tools_upscale_preparation.py index 2a8e13a4d..68ff7f350 100644 --- a/tests/test_tools_upscale_preparation.py +++ b/tests/test_tools_upscale_preparation.py @@ -1,11 +1,13 @@ """Provider-free source and processor preparation checks for Tools upscale.""" from copy import deepcopy +import hashlib import pytest from fastapi import HTTPException from PIL import Image +from services.asset_manifest import build_asset_manifest, write_asset_manifest from services.tools_upscale_commands import ( _canonical_request_source, _request_ready_params, @@ -117,6 +119,98 @@ def test_image_source_is_confined_inspected_and_snapshotted(prepared_fixture): assert resources[0]["size_bytes"] == prepared_fixture["source"].stat().st_size +def test_source_workspace_is_forwarded_and_must_match_resolved_source(prepared_fixture): + seen = {} + + def resolver(request, **_kwargs): + seen.update(request) + return ( + str(prepared_fixture["source"]), prepared_fixture["source"].name, + "source", "image", "asset-poster", "destination", + str(prepared_fixture["roots"]["destination"]), + ) + + native, _ = _prepare( + prepared_fixture, + _params(prepared_fixture, source_workspace="source"), + resolve_source=resolver, + ) + + assert seen["source_workspace"] == "source" + assert native["source_workspace"] == "source" + + with pytest.raises(HTTPException, match="source_workspace"): + _prepare( + prepared_fixture, + _params(prepared_fixture, source_workspace="destination"), + resolve_source=resolver, + ) + + +def test_asset_scope_preflight_rejects_ambiguity_and_keeps_explicit_identity(prepared_fixture): + destination = prepared_fixture["roots"]["destination"] / "poster.png" + Image.new("RGB", (19, 13), "white").save(destination) + for path, workspace in ((prepared_fixture["source"], "source"), (destination, "destination")): + write_asset_manifest( + path, + build_asset_manifest(path, asset_id="asset-shared", workspace_id=workspace, tool="fixture"), + ) + calls = [] + + def native_resolver(body, **_kwargs): + calls.append(deepcopy(body)) + path = prepared_fixture["source"] if body["source_workspace"] == "source" else destination + return str(path), path.name, body["source_workspace"], "image", "asset-shared", "destination", str( + prepared_fixture["roots"]["destination"] + ) + + resolve = _resolve_source({ + "_resolve_tool_source": native_resolver, + "_tool_asset_roots": lambda: [ + {"workspace_id": name, "path": str(path)} + for name, path in prepared_fixture["roots"].items() + ], + }) + request = { + "source": "/api/v1/assets/asset-shared", + "source_kind": "image", + "workspace": "destination", + } + with pytest.raises(HTTPException) as ambiguous: + resolve(request) + assert ambiguous.value.status_code == 409 + assert ambiguous.value.detail["code"] == "ambiguous_source" + assert calls == [] + + with pytest.raises(HTTPException) as missing: + resolve({**request, "source_workspace": "missing"}) + assert missing.value.status_code == 409 + assert missing.value.detail["code"] == "source_workspace_mismatch" + assert calls == [] + + native, resources = _prepare( + prepared_fixture, + _params( + prepared_fixture, + source="/api/v1/assets/asset-shared", + source_workspace="source", + ), + resolve_source=resolve, + ) + assert calls[-1] == { + "source_kind": "image", + "workspace": "destination", + "asset_id": "asset-shared", + "source_workspace": "source", + } + assert native["source_workspace"] == "source" + assert native["source_asset_id"] == "asset-shared" + source_hash = hashlib.sha256(prepared_fixture["source"].read_bytes()).hexdigest() + destination_hash = hashlib.sha256(destination.read_bytes()).hexdigest() + assert source_hash != destination_hash + assert resources[0]["sha256"] == source_hash + + def test_processor_reference_is_resolved_before_validator_and_keeps_order(prepared_fixture): seen = {} diff --git a/tests/test_tools_upscale_spec.py b/tests/test_tools_upscale_spec.py index 521bfe698..107893c02 100644 --- a/tests/test_tools_upscale_spec.py +++ b/tests/test_tools_upscale_spec.py @@ -49,6 +49,40 @@ def test_fingerprint_excludes_intent_but_includes_source_kind_and_method(): assert first["fingerprint"] != different_kind["fingerprint"] +def test_source_workspace_is_preserved_and_fingerprinted(): + source = "/api/v1/assets/asset-shared" + source_copy = freeze_tools_upscale_spec(command(source=source, source_workspace="source")) + other_scope = freeze_tools_upscale_spec( + command(intent="other-scope", source=source, source_workspace="default") + ) + + assert source_copy["effective"]["input"]["params"]["source_workspace"] == "source" + assert source_copy["original"]["input"]["params"]["source_workspace"] == "source" + assert source_copy["fingerprint"] != other_scope["fingerprint"] + + +@pytest.mark.parametrize("source,source_workspace", [ + ("/api/v1/file/poster.png?workspace=source", "default"), + ("/api/v1/uploads/poster.png", "source"), +]) +def test_source_workspace_must_match_canonical_url(source, source_workspace): + with pytest.raises(ToolsUpscaleSpecError, match="source_workspace"): + freeze_tools_upscale_spec(command(source=source, source_workspace=source_workspace)) + + +@pytest.mark.parametrize("source_workspace", ["__uploads__", "source", "default"]) +def test_source_workspace_accepts_known_scope_shapes(source_workspace): + source = "/api/v1/uploads/poster.png" if source_workspace == "__uploads__" else "asset-poster" + frozen = freeze_tools_upscale_spec(command(source=source, source_workspace=source_workspace)) + assert frozen["effective"]["input"]["params"]["source_workspace"] == source_workspace + + +@pytest.mark.parametrize("source_workspace", ["", " ", "../source", "/tmp/source", "s" * 161, True, 12]) +def test_source_workspace_is_strict_and_bounded(source_workspace): + with pytest.raises(ToolsUpscaleSpecError, match="source_workspace"): + freeze_tools_upscale_spec(command(source_workspace=source_workspace)) + + def test_collection_identity_is_preserved_and_fingerprinted(): first_command = command() first_command["input"]["workspace_collection_id"] = "collection-one" @@ -142,7 +176,7 @@ def test_schema_publishes_only_the_closed_tools_surface(): assert schema["operation"] == "tools.upscale" assert params["additionalProperties"] is False assert set(schema["supported_input_fields"]) == { - "workspace", "workspace_collection_id", "source", "source_kind", "method", "seed", + "workspace", "workspace_collection_id", "source", "source_workspace", "source_kind", "method", "seed", "wangp_processor_settings", } assert "actor" in schema["excluded"] From 2615bb7f7db3fcb816d65c3ae7b4bb585ee4eed5 Mon Sep 17 00:00:00 2001 From: IAnMove <216241348+IAnMove@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:50:46 +0200 Subject: [PATCH 10/59] fix(tools): avoid global asset discovery for scoped file URLs --- app/services/tools_upscale_commands.py | 2 +- tests/test_tools_upscale_preparation.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/app/services/tools_upscale_commands.py b/app/services/tools_upscale_commands.py index cb28fbbbd..bcc420eea 100644 --- a/app/services/tools_upscale_commands.py +++ b/app/services/tools_upscale_commands.py @@ -112,7 +112,7 @@ def resolve(params, **kwargs): if asset_id: body["asset_id"] = asset_id body.pop("source", None) - _check_asset_scope(runtime, asset_id, body.get("source_workspace")) + _check_asset_scope(runtime, asset_id, body.get("source_workspace")) expected_kinds = kwargs.get("expected_kinds") or (body.get("source_kind"),) try: return resolver(body, expected_kinds=expected_kinds) diff --git a/tests/test_tools_upscale_preparation.py b/tests/test_tools_upscale_preparation.py index 68ff7f350..6af2d6e0a 100644 --- a/tests/test_tools_upscale_preparation.py +++ b/tests/test_tools_upscale_preparation.py @@ -341,6 +341,21 @@ def resolver(body, **kwargs): )] +def test_scoped_file_source_does_not_depend_on_global_asset_discovery(): + def unavailable_catalog(): + raise OSError("An unrelated catalog root is unavailable") + + request = { + "source": "/api/v1/file/poster.png?workspace=source", + "source_kind": "image", "workspace": "destination", + } + resolve = _resolve_source({ + "_resolve_tool_source": lambda body, **_kwargs: body, + "_tool_asset_roots": unavailable_catalog, + }) + assert resolve(request) == request + + def test_native_projection_keeps_managed_asset_id_with_canonical_source(tmp_path): source_root = tmp_path / "source" output_root = tmp_path / "destination" From 1651fc6ec4e98573ede43b747ef8bfc9774f07b5 Mon Sep 17 00:00:00 2001 From: IAnMove <216241348+IAnMove@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:55:16 +0200 Subject: [PATCH 11/59] feat(tools): add durable upscale UI command --- scripts/export_tools_command_catalog.py | 37 ++ ui/src/api/toolsCommandCatalog.json | 491 ++++++++++++++++++ ui/src/api/toolsGenerationCommands.ts | 89 ++++ ui/src/components/Sidebar/ToolsPanel.tsx | 36 +- ui/src/features/agent/agentActionTypes.ts | 1 + ui/src/features/agent/agentActions.ts | 5 +- ui/src/features/agent/applicationAdapters.ts | 3 +- ui/src/features/agent/toolCapabilities.ts | 111 ++++ ui/src/features/agent/toolsAdapter.ts | 140 ++++- ui/src/features/studio/ToolsCommandPanel.tsx | 193 +++++++ .../studio/toolsCommandPresentation.ts | 74 +++ .../features/studio/toolsCommandSubmission.ts | 140 +++++ ui/src/features/studio/toolsGenerationSpec.ts | 296 +++++++++++ ui/src/features/studio/toolsSource.ts | 127 +++++ ui/src/i18n/locales/en/studio.json | 12 + ui/src/i18n/locales/en/wizard.json | 8 + ui/src/i18n/locales/es/studio.json | 12 + ui/src/i18n/locales/es/wizard.json | 8 + ui/src/stores/useStore.ts | 24 +- ui/tests/toolsGenerationCommands.test.ts | 267 ++++++++++ ui/tests/toolsPanel.test.tsx | 54 +- ui/tests/toolsUpscaleAdapter.test.ts | 132 +++++ 22 files changed, 2232 insertions(+), 28 deletions(-) create mode 100644 scripts/export_tools_command_catalog.py create mode 100644 ui/src/api/toolsCommandCatalog.json create mode 100644 ui/src/api/toolsGenerationCommands.ts create mode 100644 ui/src/features/studio/ToolsCommandPanel.tsx create mode 100644 ui/src/features/studio/toolsCommandPresentation.ts create mode 100644 ui/src/features/studio/toolsCommandSubmission.ts create mode 100644 ui/src/features/studio/toolsGenerationSpec.ts create mode 100644 ui/src/features/studio/toolsSource.ts create mode 100644 ui/tests/toolsGenerationCommands.test.ts create mode 100644 ui/tests/toolsUpscaleAdapter.test.ts diff --git a/scripts/export_tools_command_catalog.py b/scripts/export_tools_command_catalog.py new file mode 100644 index 000000000..85fae9924 --- /dev/null +++ b/scripts/export_tools_command_catalog.py @@ -0,0 +1,37 @@ +"""Export the executable Tools upscale contract consumed by the browser client.""" +from pathlib import Path +import argparse +import json +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "app")) +from routers.tools_upscale_commands import tools_upscale_command_catalog +from services.tools_upscale_spec import tools_upscale_schema + + +def catalog(): + return { + "version": 2, + "operations": [tools_upscale_command_catalog()], + "studio": tools_upscale_schema(), + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + path = ROOT / "ui/src/api/toolsCommandCatalog.json" + expected = json.dumps(catalog(), indent=2, ensure_ascii=False) + "\n" + if args.check: + if not path.is_file() or path.read_text(encoding="utf-8") != expected: + raise SystemExit( + "Tools command projection is stale; run scripts/export_tools_command_catalog.py and review its diff" + ) + else: + path.write_text(expected, encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/ui/src/api/toolsCommandCatalog.json b/ui/src/api/toolsCommandCatalog.json new file mode 100644 index 000000000..20d2e07ff --- /dev/null +++ b/ui/src/api/toolsCommandCatalog.json @@ -0,0 +1,491 @@ +{ + "version": 2, + "operations": [ + { + "name": "tools.upscale", + "version": 2, + "supportedVersions": [ + 2 + ], + "domain": "tools", + "mutation": true, + "description": "Upscale one exact image or video source in an explicit output workspace with an installed local processor. Preserve the source workspace, processor settings and intent_id; the shared receipt proves admission and its canonical task reports completion.", + "inputSchema": { + "type": "object", + "additionalProperties": false, + "$defs": { + "ToolsUpscaleParams": { + "additionalProperties": false, + "description": "Typed native input for one image or video upscale.", + "properties": { + "source": { + "maxLength": 8192, + "minLength": 1, + "title": "Source", + "type": "string" + }, + "source_workspace": { + "anyOf": [ + { + "maxLength": 160, + "minLength": 1, + "pattern": "^(?:__uploads__|default|[A-Za-z0-9][A-Za-z0-9_-]*)$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Workspace" + }, + "source_kind": { + "enum": [ + "image", + "video" + ], + "title": "Source Kind", + "type": "string" + }, + "method": { + "maxLength": 80, + "minLength": 1, + "title": "Method", + "type": "string" + }, + "seed": { + "default": -1, + "maximum": 9223372036854775807, + "minimum": -9223372036854775808, + "title": "Seed", + "type": "integer" + }, + "wangp_processor_settings": { + "anyOf": [ + { + "$ref": "#/$defs/WangpProcessorSettings" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "required": [ + "source", + "source_kind", + "method" + ], + "title": "ToolsUpscaleParams", + "type": "object" + }, + "WangpProcessorSettings": { + "additionalProperties": false, + "description": "Typed settings currently declared by image-capable processors.", + "properties": { + "spatial_upsampler_strength": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Spatial Upsampler Strength" + }, + "spatial_upsampler_face_count": { + "anyOf": [ + { + "maximum": 5, + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Spatial Upsampler Face Count" + }, + "spatial_upsampler_h3_strength": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Spatial Upsampler H3 Strength" + }, + "spatial_upsampler_prompt": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Spatial Upsampler Prompt" + }, + "spatial_upsampler_reference_images": { + "anyOf": [ + { + "items": { + "maxLength": 8192, + "minLength": 1, + "type": "string" + }, + "maxItems": 64, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Spatial Upsampler Reference Images" + }, + "spatial_upsampler_dlss_strength": { + "anyOf": [ + { + "maximum": 2, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Spatial Upsampler Dlss Strength" + } + }, + "title": "WangpProcessorSettings", + "type": "object" + } + }, + "properties": { + "version": { + "type": "integer", + "const": 2 + }, + "operation": { + "const": "tools.upscale" + }, + "intent_id": { + "type": "string", + "minLength": 1, + "maxLength": 160 + }, + "input": { + "additionalProperties": false, + "properties": { + "workspace": { + "maxLength": 240, + "minLength": 1, + "pattern": "^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$", + "title": "Workspace", + "type": "string" + }, + "workspace_collection_id": { + "anyOf": [ + { + "maxLength": 200, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Workspace Collection Id" + }, + "params": { + "$ref": "#/$defs/ToolsUpscaleParams" + } + }, + "required": [ + "workspace", + "params" + ], + "title": "ToolsUpscaleInput", + "type": "object" + } + }, + "required": [ + "version", + "operation", + "intent_id", + "input" + ] + } + } + ], + "studio": { + "version": 2, + "operation": "tools.upscale", + "intent_id": { + "type": "string", + "minLength": 1, + "maxLength": 160 + }, + "input": { + "$defs": { + "ToolsUpscaleParams": { + "additionalProperties": false, + "description": "Typed native input for one image or video upscale.", + "properties": { + "source": { + "maxLength": 8192, + "minLength": 1, + "title": "Source", + "type": "string" + }, + "source_workspace": { + "anyOf": [ + { + "maxLength": 160, + "minLength": 1, + "pattern": "^(?:__uploads__|default|[A-Za-z0-9][A-Za-z0-9_-]*)$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Source Workspace" + }, + "source_kind": { + "enum": [ + "image", + "video" + ], + "title": "Source Kind", + "type": "string" + }, + "method": { + "maxLength": 80, + "minLength": 1, + "title": "Method", + "type": "string" + }, + "seed": { + "default": -1, + "maximum": 9223372036854775807, + "minimum": -9223372036854775808, + "title": "Seed", + "type": "integer" + }, + "wangp_processor_settings": { + "anyOf": [ + { + "$ref": "#/$defs/WangpProcessorSettings" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "required": [ + "source", + "source_kind", + "method" + ], + "title": "ToolsUpscaleParams", + "type": "object" + }, + "WangpProcessorSettings": { + "additionalProperties": false, + "description": "Typed settings currently declared by image-capable processors.", + "properties": { + "spatial_upsampler_strength": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Spatial Upsampler Strength" + }, + "spatial_upsampler_face_count": { + "anyOf": [ + { + "maximum": 5, + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Spatial Upsampler Face Count" + }, + "spatial_upsampler_h3_strength": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Spatial Upsampler H3 Strength" + }, + "spatial_upsampler_prompt": { + "anyOf": [ + { + "maxLength": 200000, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Spatial Upsampler Prompt" + }, + "spatial_upsampler_reference_images": { + "anyOf": [ + { + "items": { + "maxLength": 8192, + "minLength": 1, + "type": "string" + }, + "maxItems": 64, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Spatial Upsampler Reference Images" + }, + "spatial_upsampler_dlss_strength": { + "anyOf": [ + { + "maximum": 2, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Spatial Upsampler Dlss Strength" + } + }, + "title": "WangpProcessorSettings", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "workspace": { + "maxLength": 240, + "minLength": 1, + "pattern": "^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$", + "title": "Workspace", + "type": "string" + }, + "workspace_collection_id": { + "anyOf": [ + { + "maxLength": 200, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Workspace Collection Id" + }, + "params": { + "$ref": "#/$defs/ToolsUpscaleParams" + } + }, + "required": [ + "workspace", + "params" + ], + "title": "ToolsUpscaleInput", + "type": "object" + }, + "supported_input_fields": [ + "workspace", + "workspace_collection_id", + "source", + "source_workspace", + "source_kind", + "method", + "seed", + "wangp_processor_settings" + ], + "source_kinds": [ + "image", + "video" + ], + "methods": [ + "dlss5*1", + "dlss5*1.5", + "dlss5*1.724", + "dlss5*2", + "dlss5*3", + "dlssg*2", + "dlssg*3", + "dlssg*4", + "dlssg*5", + "dlssg*6", + "flashvsr2", + "flashvsr2pass2", + "flashvsr2pass4", + "flashvsr3", + "flashvsr4", + "h3facerefine", + "lanczos1.5", + "lanczos2", + "rife2", + "rife3", + "rife4" + ], + "effects": { + "seed": -1, + "wangp_processor_settings": {} + }, + "excluded": [ + "actor", + "client", + "permission", + "provenance", + "workspace inside input.params", + "filesystem paths", + "remote URLs", + "free-form processor settings", + "generation.image", + "generation.speech" + ] + } +} diff --git a/ui/src/api/toolsGenerationCommands.ts b/ui/src/api/toolsGenerationCommands.ts new file mode 100644 index 000000000..5e74f4b65 --- /dev/null +++ b/ui/src/api/toolsGenerationCommands.ts @@ -0,0 +1,89 @@ +import { + createGenerationCommandClient, + GenerationCommandError, + type GenerationCommandErrorOptions, + type GenerationReceiptLike, + type GenerationTaskResult, + type SubmitGenerationCommandOptions, +} from './generationCommandClient' +import { + buildStudioToolsUpscaleGenerationCommand, + createStudioToolsUpscaleGenerationCommand, + detachedToolsUpscaleGenerationCommand, + TOOLS_UPSCALE_OPERATION, + TOOLS_UPSCALE_SCHEMA_VERSION, + type ToolsUpscaleGenerationCommand, +} from '../features/studio/toolsGenerationSpec' + +export { + assertToolsUpscaleGenerationCommand, + buildStudioToolsUpscaleGenerationCommand, + canonicalToolsSource, + createStudioToolsUpscaleGenerationCommand, + detachedToolsUpscaleGenerationCommand, + TOOLS_UPSCALE_OPERATION, + TOOLS_UPSCALE_SCHEMA_VERSION, +} from '../features/studio/toolsGenerationSpec' +export type { + ToolsUpscaleGenerationCommand, + ToolsUpscaleGenerationFullParams, + ToolsUpscaleGenerationInput, + ToolsUpscaleParamKey, + ToolsUpscaleParams, +} from '../features/studio/toolsGenerationSpec' + +export type ToolsUpscaleTaskResult = GenerationTaskResult + +export interface ToolsUpscaleGenerationReceipt extends GenerationReceiptLike { + version: 1 + operation: typeof TOOLS_UPSCALE_OPERATION + result: ToolsUpscaleTaskResult +} + +export class ToolsUpscaleGenerationCommandError extends GenerationCommandError { + constructor( + message: string, + intentId: string, + workspace: string, + options: GenerationCommandErrorOptions = {}, + ) { + super(message, intentId, workspace, { + ...options, + code: options.code ?? 'tools_upscale_command_failed', + }) + this.name = 'ToolsUpscaleGenerationCommandError' + } +} + +const toolsUpscaleCommandClient = createGenerationCommandClient< + ToolsUpscaleGenerationCommand, + ToolsUpscaleGenerationReceipt +>({ + storagePrefix: 'hocuspocus.generation.tools-upscale-commands.v2:', + contextStoragePrefix: 'hocuspocus.generation.tools-upscale-command-context.v1:', + pendingChangedEvent: 'hocuspocus:generation-tools-upscale-commands-changed', + operation: TOOLS_UPSCALE_OPERATION, + label: 'Tools upscale', + receiptFallbackVersion: TOOLS_UPSCALE_SCHEMA_VERSION, + detach: detachedToolsUpscaleGenerationCommand, + errorClass: ToolsUpscaleGenerationCommandError, + castReceipt: value => value as ToolsUpscaleGenerationReceipt, +}) + +export const newToolsUpscaleGenerationIntentId = toolsUpscaleCommandClient.newIntentId +export const pendingToolsUpscaleGenerationCommands = toolsUpscaleCommandClient.pendingCommands +export const pendingToolsUpscaleGenerationCommand = toolsUpscaleCommandClient.pendingCommand + +export type SubmitToolsUpscaleGenerationCommandOptions = SubmitGenerationCommandOptions + +export const submitToolsUpscaleGenerationCommand = toolsUpscaleCommandClient.submit +export const fetchToolsUpscaleGenerationCommandReceipt = toolsUpscaleCommandClient.fetchReceipt +export const getToolsUpscaleGenerationCommandReceipt = fetchToolsUpscaleGenerationCommandReceipt + +export function subscribeToolsUpscaleGenerationCommands(callback: () => void): () => void { + return toolsUpscaleCommandClient.subscribe(callback) +} + +/** Symmetric short aliases used by Tools callers. */ +export const createToolsUpscaleCommand = createStudioToolsUpscaleGenerationCommand +export const buildToolsUpscaleCommand = buildStudioToolsUpscaleGenerationCommand diff --git a/ui/src/components/Sidebar/ToolsPanel.tsx b/ui/src/components/Sidebar/ToolsPanel.tsx index cee9ec605..43b3c3f04 100644 --- a/ui/src/components/Sidebar/ToolsPanel.tsx +++ b/ui/src/components/Sidebar/ToolsPanel.tsx @@ -1,4 +1,4 @@ -import { useEffect, useMemo, useState } from 'react' +import { lazy, Suspense, useEffect, useMemo, useState } from 'react' import { Wrench, Play } from 'lucide-react' import { useStore } from '../../stores/useStore' import { useUiTranslation } from '../../i18n' @@ -9,6 +9,11 @@ import { catalogItemToOutput, voiceRefFromOutput } from '../../features/asset-pi import { ToolsParamsPanel } from './ToolsParamsPanel' import { resolveToolSource } from '../../lib/toolSource' import { ToolsSourcePanel } from './ToolsSourcePanel' +import { canonicalToolsSource } from '../../features/studio/toolsSource' + +const ToolsCommandPanel = lazy(() => import('../../features/studio/ToolsCommandPanel').then(module => ({ + default: module.ToolsCommandPanel, +}))) export function ToolsPanel() { const { t } = useUiTranslation('studio') @@ -113,6 +118,18 @@ export function ToolsPanel() { (tool === 'revoice' && hasVideoSource && hasRefs) || (tool === 'remove_background' && hasImageSource) const flashvsrOff = flashvsrMode === 0 && method.startsWith('flashvsr') + const commandSource = (() => { + if (sourceAssetId) return sourceAssetId + if (!sourcePath && !sourceUrl) return '' + try { + return canonicalToolsSource(sourcePath, sourceUrl, sourceWorkspace, activeWorkspace) + } catch { + // The command panel displays the malformed value so the durable builder + // can reject it with a field-specific error instead of sending legacy + // paths to the old Tools endpoint. + return sourceUrl || sourcePath || '' + } + })() return (
@@ -173,6 +190,23 @@ export function ToolsPanel() { setRemoveBackgroundInstruction={setRemoveBackgroundInstruction} /> + + { + await useStore.getState().reconnectJobs() + if (useStore.getState().activeWorkspace === receipt.result.workspace) { + await useStore.getState().maybeRefreshGallery() + } + }} + /> + + {/* Run */}