diff --git a/docs/development/PROCEDURAL_3D_SCENE_SPEC.md b/docs/development/PROCEDURAL_3D_SCENE_SPEC.md index ed8a6e08..f89ccef3 100644 --- a/docs/development/PROCEDURAL_3D_SCENE_SPEC.md +++ b/docs/development/PROCEDURAL_3D_SCENE_SPEC.md @@ -1,9 +1,11 @@ # Procedural 3D scene spec (G3) -Status: design only. No engine, UI, or `_launch_runtime.py` change ships with -this document. Playback and HTTP inspection remain blocked on the catalog -resolver contract (GROK-API-001, mailbox) and on explicit ownership of -compositor hotspots. +Status: the 2.5D compositor stays the **Video 2.5D** studio tab (`scene3d`). +Real 3D is a **separate Video 3D** tab (`world3d`) in `ui/src/features/scene3d/`. +They share asset pickers and GLB clip identity, not one overloaded panel. +Wizard hook: `hocuspocus:world3d-workflow-request` mounts the same template ids +later; 2.5D `video_3d` actions stay on the compositor. Private Meshy GLBs stay +off git. `_launch_runtime.py` is still not mounted for G2. Related: [GLB animation import contract](GLB_ANIMATION_IMPORT_CONTRACT.md) (`inspect_glb`, schema `glb-inspection-v1`). @@ -44,11 +46,13 @@ selector must use that pair. Human labels must not rename clips. | B | Add a **separate scene mode** with a real graph (nodes, camera, light, clips by id) beside the compositor | Yes | Medium | Proposed minimum. | | C | Replace the compositor with a general 3D engine | No | High | Rejected. Breaks Character Kits, atmospheres, 2.5D recipes, browser capture. | -### Decision (proposed, not implemented) +### Decision -Ship **B** only after G2’s resolver contract and hotspot ownership are -agreed. Mode A remains the default for existing templates and the Omarchy -compositor work owned by the principal. +Ship **B** as a sibling, not a replacement. The 2.5D compositor stays the +**Video 2.5D** tab. The 3D stage lives in `ui/src/features/scene3d/` as the +**Video 3D** tab (`world3d`), not a toggle inside the compositor panel. G2’s +resolver remains unmounted; local GLB files are read in the browser only. +Mode A remains the path for existing templates and the Omarchy compositor work. Criteria for choosing B for a shot: more than one mesh must share space, occlusion, a camera move in world units, or a light. Otherwise stay on A. @@ -119,6 +123,19 @@ the G1 report or the resolver, for example: No magic aliases (`idle`, `run`, `dance`) unless that exact string is the clip name. +The 2.5D compositor’s `seamlessHorizontal` / cylinder panorama has a 3D +sibling: an image slot with `loop: { cylinder: true, speed }` maps onto the +inside of an open Y-up cylinder (`CylinderGeometry`, BackSide, +RepeatWrapping). UV `offset.x = (t * speed) mod 1` scrolls the world. The +`run-loop` template is the treadmill shot: `subject_1` stays at the origin, +camera family `pursuit`, and the background cylinder moves. Clip selection +stays `(index, exact glTF name)`; do not infer Running. Any image background +can enable the same infinite cylinder without switching template. + +Video 3D export is independent of the 2.5D compositor. The world editor paints +three.js frames (`paintWorld`) and encodes H.264 in the browser. It does not +call `SceneAnimatorPanel` recording. Published files are named `world3d-*`. + ## 6. Time, persistence, languages - Scene time is deterministic. Reuse offset/speed/trim/loop/reverse from diff --git a/ui/package-lock.json b/ui/package-lock.json index ffd30bcb..3f9904ba 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -19,6 +19,7 @@ "react": "^19.2.0", "react-dom": "^19.2.0", "react-i18next": "^17.0.13", + "three": "^0.183.0", "zustand": "^5.0.11" }, "devDependencies": { @@ -30,6 +31,7 @@ "@types/node": "^24.10.1", "@types/react": "^19.2.7", "@types/react-dom": "^19.2.3", + "@types/three": "^0.183.0", "@vitejs/plugin-react": "^5.1.1", "eslint": "^9.39.1", "eslint-plugin-react-hooks": "^7.0.1", @@ -470,6 +472,13 @@ "node": ">=18" } }, + "node_modules/@dimforge/rapier3d-compat": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@dimforge/rapier3d-compat/-/rapier3d-compat-0.12.0.tgz", + "integrity": "sha512-uekIGetywIgopfD97oDL5PfeezkFpNhwlzlaEYNOA0N6ghdsOvh/HYjSMek5Q2O1PYvRSDFcqFVJl4r4ZBwOow==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.28.1", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", @@ -1972,6 +1981,13 @@ } } }, + "node_modules/@tweenjs/tween.js": { + "version": "23.1.3", + "resolved": "https://registry.npmjs.org/@tweenjs/tween.js/-/tween.js-23.1.3.tgz", + "integrity": "sha512-vJmvvwFxYuGnF2axRtPYocag6Clbb5YS7kLL+SO/TeVFzHqDIWrNKYtcsPMibjDx9O+bu+psAy9NKfWklassUA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/aria-query": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", @@ -2098,12 +2114,42 @@ "@types/react": "^19.2.0" } }, + "node_modules/@types/stats.js": { + "version": "0.17.4", + "resolved": "https://registry.npmjs.org/@types/stats.js/-/stats.js-0.17.4.tgz", + "integrity": "sha512-jIBvWWShCvlBqBNIZt0KAshWpvSjhkwkEu4ZUcASoAvhmrgAUI2t1dXrjSL4xXVLB4FznPrIsX3nKXFl/Dt4vA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/three": { + "version": "0.183.0", + "resolved": "https://registry.npmjs.org/@types/three/-/three-0.183.0.tgz", + "integrity": "sha512-AaGkvloQhxdrfMm/HbY8cpOz1K1jkEELn6zjFoY3yhAiC7zhhZE19+gDBybYwKk+GqXmWKxqDCB43NqyW3+QZw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@dimforge/rapier3d-compat": "~0.12.0", + "@tweenjs/tween.js": "~23.1.3", + "@types/stats.js": "*", + "@types/webxr": ">=0.5.17", + "@webgpu/types": "*", + "fflate": "~0.8.2", + "meshoptimizer": "~1.0.1" + } + }, "node_modules/@types/trusted-types": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", "license": "MIT" }, + "node_modules/@types/webxr": { + "version": "0.5.24", + "resolved": "https://registry.npmjs.org/@types/webxr/-/webxr-0.5.24.tgz", + "integrity": "sha512-h8fgEd/DpoS9CBrjEQXR+dIDraopAEfu4wYVNY2tEPwk60stPWhvZMf4Foo5FakuQ7HFZoa8WceaWFervK2Ovg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/wicg-file-system-access": { "version": "2020.9.8", "resolved": "https://registry.npmjs.org/@types/wicg-file-system-access/-/wicg-file-system-access-2020.9.8.tgz", @@ -2426,6 +2472,13 @@ "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" } }, + "node_modules/@webgpu/types": { + "version": "0.1.72", + "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.72.tgz", + "integrity": "sha512-0cF7RFM2edNoiIS1ODJp0/Gzv4/xSXhwoR0YCza+OWpJWtn4wmo9DvK91aLlH9+uUnwIriP7ZiC3WitmyhuzBw==", + "dev": true, + "license": "BSD-3-Clause" + }, "node_modules/acorn": { "version": "8.16.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", @@ -4086,6 +4139,13 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, + "node_modules/meshoptimizer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/meshoptimizer/-/meshoptimizer-1.0.1.tgz", + "integrity": "sha512-Vix+QlA1YYT3FwmBBZ+49cE5y/b+pRrcXKqGpS5ouh33d3lSp2PoTpCw19E0cKDFWalembrHnIaZetf27a+W2g==", + "dev": true, + "license": "MIT" + }, "node_modules/minimatch": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", @@ -4786,11 +4846,10 @@ } }, "node_modules/three": { - "version": "0.183.2", - "resolved": "https://registry.npmjs.org/three/-/three-0.183.2.tgz", - "integrity": "sha512-di3BsL2FEQ1PA7Hcvn4fyJOlxRRgFYBpMTcyOgkwJIaDOdJMebEFPA+t98EvjuljDx4hNulAGwF6KIjtwI5jgQ==", - "license": "MIT", - "peer": true + "version": "0.183.0", + "resolved": "https://registry.npmjs.org/three/-/three-0.183.0.tgz", + "integrity": "sha512-G6SH2jfefIVa2YI4JL2VbgQhrrbp1A8dRc7lr3PW827kdVyaX2RgH6M5FmjmdVFLgSHppyg3OYOZdTfWElle+g==", + "license": "MIT" }, "node_modules/tinyglobby": { "version": "0.2.15", diff --git a/ui/package.json b/ui/package.json index 47261b73..5f96c727 100644 --- a/ui/package.json +++ b/ui/package.json @@ -33,6 +33,7 @@ "react": "^19.2.0", "react-dom": "^19.2.0", "react-i18next": "^17.0.13", + "three": "^0.183.0", "zustand": "^5.0.11" }, "devDependencies": { @@ -44,6 +45,7 @@ "@types/node": "^24.10.1", "@types/react": "^19.2.7", "@types/react-dom": "^19.2.3", + "@types/three": "^0.183.0", "@vitejs/plugin-react": "^5.1.1", "eslint": "^9.39.1", "eslint-plugin-react-hooks": "^7.0.1", diff --git a/ui/src/components/MainContent/MainContent.tsx b/ui/src/components/MainContent/MainContent.tsx index 2eb4aee3..0f049a9d 100644 --- a/ui/src/components/MainContent/MainContent.tsx +++ b/ui/src/components/MainContent/MainContent.tsx @@ -23,6 +23,8 @@ import { const SceneAnimatorPanel = lazy(() => import('../Sidebar/SceneAnimatorPanel') .then(module => ({ default: module.SceneAnimatorPanel }))) +const Scene3DEditorPanel = lazy(() => import('../../features/scene3d/Scene3DEditorPanel') + .then(module => ({ default: module.Scene3DEditorPanel }))) const RigAnimatePanel = lazy(() => import('../Sidebar/RigAnimatePanel') .then(module => ({ default: module.RigAnimatePanel }))) const ComicEditorPanel = lazy(() => import('../../features/comics/ComicEditorPanel') @@ -609,6 +611,12 @@ export function MainContent() { + ) : mediaFilter === 'world3d' ? ( +
+
+ +
+
) : mediaFilter === 'animate3d' ? (
diff --git a/ui/src/components/MainContent/TabFilter.tsx b/ui/src/components/MainContent/TabFilter.tsx index 507f4a0d..4e9489f8 100644 --- a/ui/src/components/MainContent/TabFilter.tsx +++ b/ui/src/components/MainContent/TabFilter.tsx @@ -232,6 +232,7 @@ export function TabFilter() { { value: 'comics', label: t('tabs.comics'), description: t('descriptions.comics'), icon: , action: () => openFilter('comics') }, { value: 'characters', label: t('tabs.characters'), description: t('descriptions.characters'), icon: , action: () => openFilter('characters') }, { value: 'scene3d', label: t('tabs.scene3d'), description: t('descriptions.video3d'), icon: , action: () => openFilter('scene3d') }, + { value: 'world3d', label: t('tabs.world3d'), description: t('descriptions.world3d'), icon: , action: () => openFilter('world3d') }, { value: 'animate3d', label: t('tabs.animate3d'), description: t('descriptions.animate3d'), icon: , action: () => openFilter('animate3d') }, ] const productionItems: MenuItem[] = [ diff --git a/ui/src/components/Sidebar/SceneAnimatorExplorer.tsx b/ui/src/components/Sidebar/SceneAnimatorExplorer.tsx new file mode 100644 index 00000000..d5f8992e --- /dev/null +++ b/ui/src/components/Sidebar/SceneAnimatorExplorer.tsx @@ -0,0 +1,146 @@ +import type { ApiOutput } from '../../api/outputs' +import { useUiTranslation } from '../../i18n' +import type { AssetSuitability, NarrativeAssetRole } from '../../lib/assetSuitability' +import { + NARRATIVE_SCENE_TEMPLATES, + type NarrativeSceneId, + type NarrativeSceneTemplate, + type NarrativeTemplateInput, +} from '../../lib/sceneNarrative' +import { AssetExplorerDialog, AssetPickTrigger } from '../common/AssetExplorerDialog' +import { + applyExplorerChoice, + assetsForExplorer, + explorerAllowsNone, + explorerSelectedName, + explorerTitleKey, + type AssetExplorerPurpose, + type ExplorerChoiceHandlers, +} from '../common/assetExplorer.ts' + +export function SceneAnimatorExplorer({ + purpose, + models, + media, + visuals, + audio, + names, + handlers, + onClose, +}: { + purpose: AssetExplorerPurpose | null + models: ApiOutput[] + media: ApiOutput[] + visuals: ApiOutput[] + audio: ApiOutput[] + names: { hero: string; plate: string; prop: string; foreground: string } + handlers: ExplorerChoiceHandlers + onClose: () => void +}) { + const { t } = useUiTranslation('scene3d') + return ( + { + if (purpose) applyExplorerChoice(purpose, item, handlers) + onClose() + }} + /> + ) +} + +export function SceneAnimatorNarrativeSetup({ + busy, + templateId, + template, + visuals, + media, + hero, + plate, + plateLoopReady, + prop, + foreground, + mood, + intensity, + direction, + camera, + palette, + voiceSpace, + suitability, + onTemplateId, + onOpenExplorer, + onPlateLoopReady, + onMood, + onIntensity, + onDirection, + onCamera, + onPalette, + onVoiceSpace, + onMount, +}: { + busy: boolean + templateId: NarrativeSceneId + template: NarrativeSceneTemplate + visuals: ApiOutput[] + media: ApiOutput[] + hero: string + plate: string + plateLoopReady: boolean + prop: string + foreground: string + mood: NonNullable['mood'] + intensity: 1 | 2 | 3 + direction: NonNullable['direction'] + camera: NonNullable['camera'] + palette: NonNullable['palette'] + voiceSpace: NonNullable['voiceSpace'] + suitability: (role: NarrativeAssetRole, name: string) => AssetSuitability + onTemplateId: (id: NarrativeSceneId) => void + onOpenExplorer: (purpose: AssetExplorerPurpose) => void + onPlateLoopReady: (ready: boolean) => void + onMood: (value: NonNullable['mood']) => void + onIntensity: (value: 1 | 2 | 3) => void + onDirection: (value: NonNullable['direction']) => void + onCamera: (value: NonNullable['camera']) => void + onPalette: (value: NonNullable['palette']) => void + onVoiceSpace: (value: NonNullable['voiceSpace']) => void + onMount: () => void +}) { + const { t } = useUiTranslation('scene3d') + const heroFit = suitability('hero', hero) + const plateFit = suitability('plate', plate) + const propSlot = template.assetSlots.find(slot => slot.id === 'prop') + const foregroundSlot = template.assetSlots.find(slot => slot.id === 'foreground') + return ( +
+
{t('animator.narrativeTitle')}{t('animator.narrativeMeta')}
+ +
{NARRATIVE_SCENE_TEMPLATES.map(item => )}
+

{template.description}

+ asset.name === hero)} placeholder={t('animator.chooseAsset')} disabled={busy} onOpen={() => onOpenExplorer('narrative-hero')} /> + {hero ?

{heroFit.message}

: null} + asset.name === plate)} placeholder={t('animator.chooseAsset')} disabled={busy} onOpen={() => onOpenExplorer('narrative-plate')} /> + {plate && plateFit.level !== 'ok' ?

{plateFit.message}

: null} + {plate ? : null} + {propSlot ? asset.name === prop)} placeholder={t('animator.none')} disabled={busy} onOpen={() => onOpenExplorer('narrative-prop')} /> : null} + {foregroundSlot ? asset.name === foreground)} placeholder={t('animator.none')} disabled={busy} onOpen={() => onOpenExplorer('narrative-foreground')} /> : null} +
+ {template.controls.includes('mood') ? : null} + {template.controls.includes('intensity') ? : null} + {template.controls.includes('direction') ? : null} + {template.controls.includes('camera') ? : null} + {template.controls.includes('palette') ? : null} + {template.controls.includes('voiceSpace') ? : null} +
+ +
+ ) +} diff --git a/ui/src/components/Sidebar/SceneAnimatorPanel.tsx b/ui/src/components/Sidebar/SceneAnimatorPanel.tsx index 89fabdb4..714dd756 100644 --- a/ui/src/components/Sidebar/SceneAnimatorPanel.tsx +++ b/ui/src/components/Sidebar/SceneAnimatorPanel.tsx @@ -1,10 +1,13 @@ import { memo, useCallback, useEffect, useRef, useState, type CSSProperties, type PointerEvent as ReactPointerEvent } from 'react' import type { ParseKeys } from 'i18next' -import { AlignHorizontalJustifyCenter, AlignVerticalJustifyCenter, Box, Camera, ChevronDown, ChevronDown as Down, ChevronUp, CloudRain, Copy, CopyPlus, Download, Eye, EyeOff, FileJson, Film, FolderOpen, Grid3X3, Image as ImageIcon, Loader2, Lock, Magnet, Mic, Play, Plus, Redo2, Save, Trash2, Undo2, Unlock, Video } from 'lucide-react' +import { AlignHorizontalJustifyCenter, AlignVerticalJustifyCenter, Box, Camera, ChevronDown, ChevronUp, CloudRain, Copy, CopyPlus, Download, Eye, EyeOff, FileJson, Film, FolderOpen, Grid3X3, Image as ImageIcon, Loader2, Lock, Magnet, Mic, Play, Plus, Redo2, Save, Trash2, Undo2, Unlock, Video } from 'lucide-react' import { ArrayBufferTarget, Muxer } from 'mp4-muxer' import { useUiTranslation } from '../../i18n' import { useStore } from '../../stores/useStore' import { analyzeAudio, deleteCharacterKit, fetchCharacterKitLibrary, fetchOutputs, generateLlmText, saveCharacterKit, saveScene as saveSceneOutput, saveSceneRecording, uploadImage } from '../../api/client' +import { AssetPickTrigger } from '../common/AssetExplorerDialog' +import type { AssetExplorerPurpose } from '../common/assetExplorer.ts' +import { SceneAnimatorExplorer, SceneAnimatorNarrativeSetup } from './SceneAnimatorExplorer' import { generateSceneSpeechClip } from '../../lib/sceneSpeech' import { SceneRecipePanel } from './SceneRecipePanel' import { TemplateComposerDialog } from '../../features/sceneTemplates/TemplateComposerDialog' @@ -22,7 +25,7 @@ import { applyCutoutDialogue, bindCutoutFaceToPose, ensureCutoutFacePlayback, fi import { captureCharacterFaceAnchor, characterKitAssetFromLayer, createCharacterKit, emptyCharacterKitLibrary, mountCharacterKitLayers, syncMountedCharacterKitLayers, syncSceneCharacterKits, type CharacterKit, type CharacterKitAlphaStatus, type CharacterMouthState } from '../../lib/characterKit' import { consumeFaceRigHandoff, FACE_RIG_HANDOFF_EVENT, kitFromFaceRigHandoff } from '../../lib/characterKitHandoff' import { rememberCharacterKitLibrary, rememberVideo3dScene } from '../../features/agent/wizardLabSession' -import { carrySceneSidecars, createNarrativeScene, getNarrativeTemplate, NARRATIVE_SCENE_TEMPLATES, type NarrativeSceneId, type NarrativeTemplateInput } from '../../lib/sceneNarrative' +import { carrySceneSidecars, createNarrativeScene, getNarrativeTemplate, type NarrativeSceneId, type NarrativeTemplateInput } from '../../lib/sceneNarrative' import { applySceneCopilotProposal, buildSceneCopilotSystemPrompt, buildSceneScopeCopilotSystemPrompt, describeSceneCopilotProposal, parseSceneCopilotProposal, SCENE_COPILOT_JSON_SCHEMA, type SceneCopilotProposal } from '../../lib/sceneCopilot' import { evaluateSceneLayer, getSceneEvents, getSceneKeyframes, getSceneLayerTiming, mapSceneAnimationPoints, normalizeSceneEvents, normalizeSceneKeyframes, sceneLayerMotionProgress, sceneProgressFromSeconds, sceneTimeToLayerTime, withNormalizedSceneTiming, withSceneKeyframes } from '../../lib/sceneTimeline' import { normalizeSeamOccluder, paintSeamOccluder, seamOccluderDataUri, type SeamOccluderKind } from '../../lib/seamOccluder' @@ -486,7 +489,7 @@ export function SceneAnimatorPanel() { const [selectedId, setSelectedId] = useState(null) const [addOpen, setAddOpen] = useState(false) const [templateComposerOpen, setTemplateComposerOpen] = useState(false) - const [picker, setPicker] = useState<'model' | 'media' | null>(null) + const [assetExplorer, setAssetExplorer] = useState(null) const [playing, setPlaying] = useState(false) const [recording, setRecording] = useState(false) const [publishing, setPublishing] = useState(false) @@ -571,6 +574,7 @@ export function SceneAnimatorPanel() { const [previewWidth, setPreviewWidth] = useState(1280) const [clipsByLayer, setClipsByLayer] = useState>({}) const [clipDurationsByLayer, setClipDurationsByLayer] = useState>({}) + const canvasRef = useRef(null) const animationRef = useRef(null) const recordingAnimationRef = useRef(null) @@ -901,7 +905,7 @@ export function SceneAnimatorPanel() { const layers = type === 'image' || type === 'video' ? [layer, ...ordered] : [...ordered, layer] return { ...current, layers: normalizeZ(layers) } }) - setSelectedId(id); setAddOpen(false); setPicker(null) + setSelectedId(id); setAddOpen(false); setAssetExplorer(null) } const addAtmosphere = (kind: SceneAtmosphereKind) => { const id = uid() @@ -930,7 +934,7 @@ export function SceneAnimatorPanel() { } return { ...current, layers: normalizeZ([...current.layers, layer]) } }) - setSelectedId(id); setAddOpen(false); setPicker(null) + setSelectedId(id); setAddOpen(false); setAssetExplorer(null) } const addCamera = () => { const id = uid() @@ -954,7 +958,7 @@ export function SceneAnimatorPanel() { const layers = current.layers.map(layer => layer.type === 'camera' ? { ...layer, visible: false } : layer) return { ...current, layers: normalizeZ([...layers, camera]) } }) - setSelectedId(id); setAddOpen(false); setPicker(null); setProgress(0) + setSelectedId(id); setAddOpen(false); setAssetExplorer(null); setProgress(0) } const duplicateLayer = (id: string) => { const original = sceneRef.current.layers.find(layer => layer.id === id) @@ -2978,30 +2982,35 @@ export function SceneAnimatorPanel() { />