From ec0f7846e80a428af75bd8f9dec35926ea041f43 Mon Sep 17 00:00:00 2001 From: IAnMove <216241348+IAnMove@users.noreply.github.com> Date: Sun, 6 Sep 2026 23:38:58 +0200 Subject: [PATCH 1/7] feat(video3d): add a real 3D stage beside the 2.5D compositor Video 3D keeps the layered compositor as the default. A sibling three.js stage (metres, Y-up) plays glTF clips only when index and exact name match. Local GLB files stay in the browser. WebGL is disposed on unmount, canvas size is capped, and blob URLs are revoked so the stage does not keep a second renderer alive after switching back to 2.5D. --- docs/development/PROCEDURAL_3D_SCENE_SPEC.md | 18 +- ui/package-lock.json | 69 +++++- ui/package.json | 2 + .../components/Sidebar/SceneAnimatorPanel.tsx | 15 +- ui/src/features/scene3d/Scene3DStage.tsx | 146 ++++++++++++ ui/src/features/scene3d/Scene3DWorkspace.tsx | 198 +++++++++++++++++ ui/src/features/scene3d/camera.ts | 74 +++++++ ui/src/features/scene3d/clips.ts | 30 +++ ui/src/features/scene3d/clock.ts | 28 +++ ui/src/features/scene3d/document.ts | 71 ++++++ ui/src/features/scene3d/gpu.ts | 209 ++++++++++++++++++ ui/src/features/scene3d/index.ts | 17 ++ ui/src/features/scene3d/softwareRender.ts | 62 ++++++ ui/src/features/scene3d/types.ts | 69 ++++++ ui/src/i18n/locales/en/scene3d.json | 25 +++ ui/src/i18n/locales/es/scene3d.json | 25 +++ ui/tests/scene3dStage.test.mjs | 71 ++++++ 17 files changed, 1114 insertions(+), 15 deletions(-) create mode 100644 ui/src/features/scene3d/Scene3DStage.tsx create mode 100644 ui/src/features/scene3d/Scene3DWorkspace.tsx create mode 100644 ui/src/features/scene3d/camera.ts create mode 100644 ui/src/features/scene3d/clips.ts create mode 100644 ui/src/features/scene3d/clock.ts create mode 100644 ui/src/features/scene3d/document.ts create mode 100644 ui/src/features/scene3d/gpu.ts create mode 100644 ui/src/features/scene3d/index.ts create mode 100644 ui/src/features/scene3d/softwareRender.ts create mode 100644 ui/src/features/scene3d/types.ts create mode 100644 ui/tests/scene3dStage.test.mjs diff --git a/docs/development/PROCEDURAL_3D_SCENE_SPEC.md b/docs/development/PROCEDURAL_3D_SCENE_SPEC.md index ed8a6e08..c5cc022a 100644 --- a/docs/development/PROCEDURAL_3D_SCENE_SPEC.md +++ b/docs/development/PROCEDURAL_3D_SCENE_SPEC.md @@ -1,9 +1,9 @@ # Procedural 3D scene spec (G3) -Status: design only. No engine, UI, or `_launch_runtime.py` change ships with -this document. Playback and HTTP inspection remain blocked on the catalog -resolver contract (GROK-API-001, mailbox) and on explicit ownership of -compositor hotspots. +Status: vertical in Video 3D. The 2.5D compositor stays default. A **3D stage** +mode lives in `ui/src/features/scene3d/` and is selected with a toggle in +Video 3D. Private Meshy GLBs stay off git. `_launch_runtime.py` is still not +mounted for G2. Related: [GLB animation import contract](GLB_ANIMATION_IMPORT_CONTRACT.md) (`inspect_glb`, schema `glb-inspection-v1`). @@ -44,11 +44,13 @@ selector must use that pair. Human labels must not rename clips. | B | Add a **separate scene mode** with a real graph (nodes, camera, light, clips by id) beside the compositor | Yes | Medium | Proposed minimum. | | C | Replace the compositor with a general 3D engine | No | High | Rejected. Breaks Character Kits, atmospheres, 2.5D recipes, browser capture. | -### Decision (proposed, not implemented) +### Decision -Ship **B** only after G2’s resolver contract and hotspot ownership are -agreed. Mode A remains the default for existing templates and the Omarchy -compositor work owned by the principal. +Ship **B** as a sibling, not a replacement. Video 3D keeps the 2.5D compositor +as the default (`stageMode=compositor`). The 3D stage lives in +`ui/src/features/scene3d/` and is selected with the 2.5D | 3D toggle. G2’s +resolver remains unmounted; local GLB files are read in the browser only. +Mode A remains the path for existing templates and the Omarchy compositor work. Criteria for choosing B for a shot: more than one mesh must share space, occlusion, a camera move in world units, or a light. Otherwise stay on A. diff --git a/ui/package-lock.json b/ui/package-lock.json index ffd30bcb..3f9904ba 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -19,6 +19,7 @@ "react": "^19.2.0", "react-dom": "^19.2.0", "react-i18next": "^17.0.13", + "three": "^0.183.0", "zustand": "^5.0.11" }, "devDependencies": { @@ -30,6 +31,7 @@ "@types/node": "^24.10.1", "@types/react": "^19.2.7", "@types/react-dom": "^19.2.3", + "@types/three": "^0.183.0", "@vitejs/plugin-react": "^5.1.1", "eslint": "^9.39.1", "eslint-plugin-react-hooks": "^7.0.1", @@ -470,6 +472,13 @@ "node": ">=18" } }, + "node_modules/@dimforge/rapier3d-compat": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@dimforge/rapier3d-compat/-/rapier3d-compat-0.12.0.tgz", + "integrity": "sha512-uekIGetywIgopfD97oDL5PfeezkFpNhwlzlaEYNOA0N6ghdsOvh/HYjSMek5Q2O1PYvRSDFcqFVJl4r4ZBwOow==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.28.1", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", @@ -1972,6 +1981,13 @@ } } }, + "node_modules/@tweenjs/tween.js": { + "version": "23.1.3", + "resolved": "https://registry.npmjs.org/@tweenjs/tween.js/-/tween.js-23.1.3.tgz", + "integrity": "sha512-vJmvvwFxYuGnF2axRtPYocag6Clbb5YS7kLL+SO/TeVFzHqDIWrNKYtcsPMibjDx9O+bu+psAy9NKfWklassUA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/aria-query": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", @@ -2098,12 +2114,42 @@ "@types/react": "^19.2.0" } }, + "node_modules/@types/stats.js": { + "version": "0.17.4", + "resolved": "https://registry.npmjs.org/@types/stats.js/-/stats.js-0.17.4.tgz", + "integrity": "sha512-jIBvWWShCvlBqBNIZt0KAshWpvSjhkwkEu4ZUcASoAvhmrgAUI2t1dXrjSL4xXVLB4FznPrIsX3nKXFl/Dt4vA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/three": { + "version": "0.183.0", + "resolved": "https://registry.npmjs.org/@types/three/-/three-0.183.0.tgz", + "integrity": "sha512-AaGkvloQhxdrfMm/HbY8cpOz1K1jkEELn6zjFoY3yhAiC7zhhZE19+gDBybYwKk+GqXmWKxqDCB43NqyW3+QZw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@dimforge/rapier3d-compat": "~0.12.0", + "@tweenjs/tween.js": "~23.1.3", + "@types/stats.js": "*", + "@types/webxr": ">=0.5.17", + "@webgpu/types": "*", + "fflate": "~0.8.2", + "meshoptimizer": "~1.0.1" + } + }, "node_modules/@types/trusted-types": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", "license": "MIT" }, + "node_modules/@types/webxr": { + "version": "0.5.24", + "resolved": "https://registry.npmjs.org/@types/webxr/-/webxr-0.5.24.tgz", + "integrity": "sha512-h8fgEd/DpoS9CBrjEQXR+dIDraopAEfu4wYVNY2tEPwk60stPWhvZMf4Foo5FakuQ7HFZoa8WceaWFervK2Ovg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/wicg-file-system-access": { "version": "2020.9.8", "resolved": "https://registry.npmjs.org/@types/wicg-file-system-access/-/wicg-file-system-access-2020.9.8.tgz", @@ -2426,6 +2472,13 @@ "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" } }, + "node_modules/@webgpu/types": { + "version": "0.1.72", + "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.72.tgz", + "integrity": "sha512-0cF7RFM2edNoiIS1ODJp0/Gzv4/xSXhwoR0YCza+OWpJWtn4wmo9DvK91aLlH9+uUnwIriP7ZiC3WitmyhuzBw==", + "dev": true, + "license": "BSD-3-Clause" + }, "node_modules/acorn": { "version": "8.16.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", @@ -4086,6 +4139,13 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, + "node_modules/meshoptimizer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/meshoptimizer/-/meshoptimizer-1.0.1.tgz", + "integrity": "sha512-Vix+QlA1YYT3FwmBBZ+49cE5y/b+pRrcXKqGpS5ouh33d3lSp2PoTpCw19E0cKDFWalembrHnIaZetf27a+W2g==", + "dev": true, + "license": "MIT" + }, "node_modules/minimatch": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", @@ -4786,11 +4846,10 @@ } }, "node_modules/three": { - "version": "0.183.2", - "resolved": "https://registry.npmjs.org/three/-/three-0.183.2.tgz", - "integrity": "sha512-di3BsL2FEQ1PA7Hcvn4fyJOlxRRgFYBpMTcyOgkwJIaDOdJMebEFPA+t98EvjuljDx4hNulAGwF6KIjtwI5jgQ==", - "license": "MIT", - "peer": true + "version": "0.183.0", + "resolved": "https://registry.npmjs.org/three/-/three-0.183.0.tgz", + "integrity": "sha512-G6SH2jfefIVa2YI4JL2VbgQhrrbp1A8dRc7lr3PW827kdVyaX2RgH6M5FmjmdVFLgSHppyg3OYOZdTfWElle+g==", + "license": "MIT" }, "node_modules/tinyglobby": { "version": "0.2.15", diff --git a/ui/package.json b/ui/package.json index 47261b73..5f96c727 100644 --- a/ui/package.json +++ b/ui/package.json @@ -33,6 +33,7 @@ "react": "^19.2.0", "react-dom": "^19.2.0", "react-i18next": "^17.0.13", + "three": "^0.183.0", "zustand": "^5.0.11" }, "devDependencies": { @@ -44,6 +45,7 @@ "@types/node": "^24.10.1", "@types/react": "^19.2.7", "@types/react-dom": "^19.2.3", + "@types/three": "^0.183.0", "@vitejs/plugin-react": "^5.1.1", "eslint": "^9.39.1", "eslint-plugin-react-hooks": "^7.0.1", diff --git a/ui/src/components/Sidebar/SceneAnimatorPanel.tsx b/ui/src/components/Sidebar/SceneAnimatorPanel.tsx index afdddbe8..822b3ef9 100644 --- a/ui/src/components/Sidebar/SceneAnimatorPanel.tsx +++ b/ui/src/components/Sidebar/SceneAnimatorPanel.tsx @@ -32,6 +32,7 @@ import { CylinderPanoramaComparison } from './CylinderPanoramaComparison' import { CharacterKitLibraryPanel } from '../../features/characters/CharacterKitLibraryPanel' import type { CharacterKitEditorTab } from '../../features/characters/characterKitGuide' import { listenForAgentSceneControl, listenForAgentSceneRhythm, listenForAgentSceneWorkflow } from '../../features/agent/agentUiBus' +import { Scene3DWorkspace } from '../../features/scene3d/Scene3DWorkspace' type Point = { x: number; y: number; scale: number; opacity?: number; rotation?: number } type AnimatorLayerType = SceneLayerType @@ -571,6 +572,7 @@ export function SceneAnimatorPanel() { const [previewWidth, setPreviewWidth] = useState(1280) const [clipsByLayer, setClipsByLayer] = useState>({}) const [clipDurationsByLayer, setClipDurationsByLayer] = useState>({}) + const [stageMode, setStageMode] = useState<'compositor' | 'world3d'>('compositor') const canvasRef = useRef(null) const animationRef = useRef(null) const recordingAnimationRef = useRef(null) @@ -2940,7 +2942,16 @@ export function SceneAnimatorPanel() { {selected && isVisualLayer(selected) && selected.type !== 'model3d' && selected.type !== 'effect' && } {selected && isVisualLayer(selected) && selected.type !== 'model3d' && selected.type !== 'effect' && } -
+
+ + +
+ {stageMode === 'world3d' ?
:
{[...scene.layers].sort((a, b) => a.z - b.z).map(renderLayer)} {composition.showGrid &&
} @@ -2952,7 +2963,7 @@ export function SceneAnimatorPanel() { {flash &&
}
-
+
}

{t('animator.canvasHelp')}

void +} + +function loadSlotGltf( + world: GpuWorld, + slot: Scene3DSlot, + loader: GLTFLoader, + cancelled: () => boolean, + liveSlot: () => Scene3DSlot | undefined, + onClips: ((slotId: string, clips: Scene3DClipCatalogEntry[]) => void) | undefined, +) { + const url = slot.sourceUrl + if (!url) return + loader.load( + url, + (gltf: GLTF) => { + if (cancelled()) { + disposeObject(gltf.scene) + return + } + const live = liveSlot() + if (!live || live.sourceUrl !== url) { + disposeObject(gltf.scene) + return + } + fitGltf(gltf.scene, live) + placeSlot(world, live, gltf.scene, gltf.animations) + onClips?.(live.id, catalogFromClips(gltf.animations)) + }, + undefined, + () => undefined, + ) +} + +export function Scene3DStage({ document, sceneSeconds, onSlotClips }: Props) { + const hostRef = useRef(null) + const worldRef = useRef(null) + const documentRef = useRef(document) + const onSlotClipsRef = useRef(onSlotClips) + + useEffect(() => { + documentRef.current = document + }, [document]) + + useEffect(() => { + onSlotClipsRef.current = onSlotClips + }, [onSlotClips]) + + useEffect(() => { + const host = hostRef.current + if (!host) return + const world = createWorld(host, documentRef.current.light, documentRef.current.camera.fov) + worldRef.current = world + const resize = () => resizeWorld(world, host) + resize() + const observer = new ResizeObserver(resize) + observer.observe(host) + return () => { + observer.disconnect() + disposeWorld(world) + worldRef.current = null + } + }, []) + + useEffect(() => { + const world = worldRef.current + if (!world) return + applyLight(world.dir, document.light) + }, [document.light]) + + useEffect(() => { + const world = worldRef.current + if (!world) return + let cancelled = false + const loader = new GLTFLoader() + pruneSlots(world, document.slots) + for (const slot of document.slots) { + const current = world.slots.get(slot.id) + if (current && current.sourceUrl === slot.sourceUrl) { + poseLoadedSlot(current, slot) + syncSlotClip(world, slot) + continue + } + placeSlot(world, slot, placeholderMesh(slot), []) + loadSlotGltf( + world, + slot, + loader, + () => cancelled || worldRef.current !== world, + () => documentRef.current.slots.find(item => item.id === slot.id), + (slotId, clips) => onSlotClipsRef.current?.(slotId, clips), + ) + } + return () => { + cancelled = true + } + }, [document.slots]) + + useEffect(() => { + const world = worldRef.current + if (!world) return + const eye = cameraEyeAtTime(document.camera, sceneSeconds, document.duration) + world.camera.fov = document.camera.fov + world.camera.position.set(eye[0], eye[1], eye[2]) + world.camera.lookAt(document.camera.look[0], document.camera.look[1], document.camera.look[2]) + world.camera.updateProjectionMatrix() + for (const gpu of world.slots.values()) { + if (!gpu.mixer) continue + const clip = gpu.animations.find((_, index) => clipMatches(gpu, index)) + const local = scene3dClipLocalTime(sceneSeconds, clip?.duration ?? null, { loop: true }) + if (local != null) gpu.mixer.setTime(local) + } + world.renderer.render(world.scene, world.camera) + }, [document.camera, document.duration, sceneSeconds]) + + return
+} + +function clipMatches(gpu: { clipKey: string; animations: GLTF['animations'] }, index: number) { + const clip = gpu.animations[index] + return Boolean(clip && gpu.clipKey === `${index}\0${clip.name}`) +} diff --git a/ui/src/features/scene3d/Scene3DWorkspace.tsx b/ui/src/features/scene3d/Scene3DWorkspace.tsx new file mode 100644 index 00000000..e884c237 --- /dev/null +++ b/ui/src/features/scene3d/Scene3DWorkspace.tsx @@ -0,0 +1,198 @@ +import { useEffect, useMemo, useRef, useState } from 'react' +import { useUiTranslation } from '../../i18n' +import { cameraEyeAtTime } from './camera.ts' +import { clipBindingError, resolveScene3DClip } from './clips.ts' +import { scene3dFrameCount, scene3dFrameTime } from './clock.ts' +import { createDefaultScene3DDocument, parseScene3DDocument } from './document.ts' +import { Scene3DStage } from './Scene3DStage.tsx' +import type { Scene3DClipCatalogEntry, Scene3DDocument } from './types.ts' + +const FAMILIES = ['establishment', 'orbit', 'follow', 'product'] as const +type StageFamily = (typeof FAMILIES)[number] + +type Props = { + width: number + height: number +} + +function revokeIfBlob(url: string) { + if (url.startsWith('blob:')) URL.revokeObjectURL(url) +} + +export function Scene3DWorkspace({ width, height }: Props) { + const { t } = useUiTranslation('scene3d') + const [sceneDoc, setSceneDoc] = useState(() => createDefaultScene3DDocument()) + const [playing, setPlaying] = useState(false) + const [frame, setFrame] = useState(0) + const frameRef = useRef(0) + const sceneDocRef = useRef(sceneDoc) + const [catalogs, setCatalogs] = useState>({}) + const fps = sceneDoc.fps + const count = scene3dFrameCount(sceneDoc.duration, fps) + const seconds = scene3dFrameTime(frame, sceneDoc.duration, fps) + + useEffect(() => { + frameRef.current = frame + }, [frame]) + + useEffect(() => { + sceneDocRef.current = sceneDoc + }, [sceneDoc]) + + useEffect(() => () => { + for (const slot of sceneDocRef.current.slots) revokeIfBlob(slot.sourceUrl) + }, []) + + const clipIssue = useMemo(() => { + for (const slot of sceneDoc.slots) { + const entries = catalogs[slot.id] + if (!slot.clip || entries == null) continue + const error = clipBindingError(resolveScene3DClip(entries, slot.clip)) + if (error) return `${slot.slot}: ${error.message}` + } + return null + }, [catalogs, sceneDoc.slots]) + + useEffect(() => { + if (!playing) return + let raf = 0 + const origin = performance.now() + const originFrame = frameRef.current + const tick = () => { + const elapsed = scene3dFrameTime(originFrame, sceneDoc.duration, fps) + (performance.now() - origin) / 1000 + const wrapped = elapsed % Math.max(sceneDoc.duration, 0.001) + const next = Math.min(count - 1, Math.round(wrapped * fps)) + setFrame(current => (current === next ? current : next)) + raf = requestAnimationFrame(tick) + } + raf = requestAnimationFrame(tick) + return () => cancelAnimationFrame(raf) + }, [playing, sceneDoc.duration, fps, count]) + + const assignFile = (slotId: string, file: File | undefined) => { + if (!file) return + const url = URL.createObjectURL(file) + setSceneDoc(current => ({ + ...current, + slots: current.slots.map(slot => { + if (slot.id !== slotId) return slot + revokeIfBlob(slot.sourceUrl) + return { ...slot, sourceUrl: url, clip: null } + }), + })) + setCatalogs(current => { + const next = { ...current } + delete next[slotId] + return next + }) + } + + const setFamily = (family: StageFamily) => { + setSceneDoc(current => ({ + ...current, + camera: { ...current.camera, family }, + })) + } + + const bindClip = (slotId: string, index: number, name: string) => { + setSceneDoc(current => ({ + ...current, + slots: current.slots.map(slot => ( + slot.id === slotId + ? { ...slot, clip: name ? { index, name } : null } + : slot + )), + })) + } + + const roundtrip = Boolean(parseScene3DDocument(JSON.parse(JSON.stringify(sceneDoc)))) + + return ( +
+
+ setCatalogs(current => ({ ...current, [slotId]: clips }))} + /> +
+ {t('stage.badge')} · {sceneDoc.camera.family} · {seconds.toFixed(2)}s +
+
+
+ {FAMILIES.map(family => ( + + ))} + + {t('stage.eye', { x: cameraEyeAtTime(sceneDoc.camera, seconds, sceneDoc.duration)[0].toFixed(2) })} +
+
+ {sceneDoc.slots.map(slot => { + const clips = catalogs[slot.id] ?? [] + return ( + + ) + })} +
+ {clipIssue &&

{clipIssue}

} +

{t('stage.help')}

+ {roundtrip ? 'ok' : 'bad'} +
+ ) +} diff --git a/ui/src/features/scene3d/camera.ts b/ui/src/features/scene3d/camera.ts new file mode 100644 index 00000000..acdb8177 --- /dev/null +++ b/ui/src/features/scene3d/camera.ts @@ -0,0 +1,74 @@ +import type { Scene3DCamera, Vec3 } from './types.ts' + +export function vecAdd(a: Vec3, b: Vec3): Vec3 { + return [a[0] + b[0], a[1] + b[1], a[2] + b[2]] +} + +export function vecSub(a: Vec3, b: Vec3): Vec3 { + return [a[0] - b[0], a[1] - b[1], a[2] - b[2]] +} + +export function vecScale(a: Vec3, s: number): Vec3 { + return [a[0] * s, a[1] * s, a[2] * s] +} + +export function vecDot(a: Vec3, b: Vec3): number { + return a[0] * b[0] + a[1] * b[1] + a[2] * b[2] +} + +export function vecCross(a: Vec3, b: Vec3): Vec3 { + return [a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0]] +} + +export function vecLength(a: Vec3): number { + return Math.hypot(a[0], a[1], a[2]) +} + +export function vecNormalize(a: Vec3): Vec3 { + const len = vecLength(a) + if (len < 1e-8) return [0, 1, 0] + return vecScale(a, 1 / len) +} + +export function orbitEye(look: Vec3, radius: number, height: number, azimuthRad: number): Vec3 { + const r = Number.isFinite(radius) && radius > 0 ? radius : 4 + const h = Number.isFinite(height) ? height : 1.6 + return [ + look[0] + Math.sin(azimuthRad) * r, + look[1] + h, + look[2] + Math.cos(azimuthRad) * r, + ] +} + +export function cameraEyeAtTime(camera: Scene3DCamera, sceneSeconds: number, duration: number): Vec3 { + if (camera.family !== 'orbit') return camera.eye + const turns = Number.isFinite(camera.orbitTurns) ? camera.orbitTurns! : 1 + const span = duration > 1e-6 ? duration : 1 + const azimuth = (sceneSeconds / span) * turns * Math.PI * 2 + return orbitEye(camera.look, camera.orbitRadius ?? 4, camera.orbitHeight ?? 1.6, azimuth) +} + +export function projectPoint( + point: Vec3, + eye: Vec3, + look: Vec3, + fovDeg: number, + aspect: number, +): { x: number; y: number; depth: number } | null { + const forward = vecNormalize(vecSub(look, eye)) + const worldUp: Vec3 = [0, 1, 0] + let right = vecCross(forward, worldUp) + if (vecLength(right) < 1e-6) right = [1, 0, 0] + right = vecNormalize(right) + const up = vecNormalize(vecCross(right, forward)) + const rel = vecSub(point, eye) + const camX = vecDot(rel, right) + const camY = vecDot(rel, up) + const camZ = vecDot(rel, forward) + if (camZ <= 1e-4) return null + const fov = (Number.isFinite(fovDeg) && fovDeg > 1 ? fovDeg : 50) * Math.PI / 180 + const f = 1 / Math.tan(fov / 2) + const ndcX = (camX * f) / (aspect * camZ) + const ndcY = (camY * f) / camZ + return { x: (ndcX + 1) / 2, y: 1 - (ndcY + 1) / 2, depth: camZ } +} diff --git a/ui/src/features/scene3d/clips.ts b/ui/src/features/scene3d/clips.ts new file mode 100644 index 00000000..0211222d --- /dev/null +++ b/ui/src/features/scene3d/clips.ts @@ -0,0 +1,30 @@ +import type { Scene3DClipCatalogEntry, Scene3DClipError, Scene3DClipRef } from './types.ts' + +export function resolveScene3DClip( + catalog: readonly Scene3DClipCatalogEntry[], + wanted: Scene3DClipRef | null, +): Scene3DClipCatalogEntry | Scene3DClipError | null { + if (!wanted) return null + const index = wanted.index + if (!Number.isInteger(index) || index < 0 || index >= catalog.length) { + return { + code: 'clip_missing', + message: `No clip at index ${index}`, + } + } + const found = catalog[index] + if (found.name !== wanted.name) { + return { + code: 'clip_name_mismatch', + message: `Clip ${index} is ${JSON.stringify(found.name)}, not ${JSON.stringify(wanted.name)}`, + } + } + return found +} + +export function clipBindingError( + result: Scene3DClipCatalogEntry | Scene3DClipError | null, +): Scene3DClipError | null { + if (result && 'code' in result) return result + return null +} diff --git a/ui/src/features/scene3d/clock.ts b/ui/src/features/scene3d/clock.ts new file mode 100644 index 00000000..762f19a2 --- /dev/null +++ b/ui/src/features/scene3d/clock.ts @@ -0,0 +1,28 @@ +/** Match compositor export: frameCount = Math.round(duration * fps). */ +export function scene3dFrameCount(duration: number, fps: number): number { + const safeDuration = Number.isFinite(duration) && duration > 0 ? duration : 0 + const safeFps = Number.isFinite(fps) && fps > 0 ? fps : 30 + return Math.max(1, Math.round(safeDuration * safeFps)) +} + +/** Frame i is t = min(duration, i / fps) for i = 0 .. frameCount-1. */ +export function scene3dFrameTime(index: number, duration: number, fps: number): number { + const count = scene3dFrameCount(duration, fps) + const i = Math.max(0, Math.min(count - 1, Math.trunc(index))) + const safeFps = Number.isFinite(fps) && fps > 0 ? fps : 30 + const safeDuration = Number.isFinite(duration) && duration > 0 ? duration : 0 + return Math.min(safeDuration, i / safeFps) +} + +export function scene3dClipLocalTime( + sceneSeconds: number, + clipDuration: number | null, + options: { offset?: number; speed?: number; loop?: boolean } = {}, +): number | null { + if (clipDuration == null || !Number.isFinite(clipDuration) || clipDuration <= 0) return null + const offset = Number.isFinite(options.offset) ? Math.max(0, options.offset!) : 0 + const speed = Number.isFinite(options.speed) && options.speed! > 0 ? options.speed! : 1 + const elapsed = Math.max(0, sceneSeconds - offset) * speed + if (options.loop === false) return Math.min(clipDuration, elapsed) + return elapsed % clipDuration +} diff --git a/ui/src/features/scene3d/document.ts b/ui/src/features/scene3d/document.ts new file mode 100644 index 00000000..2d0a0ab9 --- /dev/null +++ b/ui/src/features/scene3d/document.ts @@ -0,0 +1,71 @@ +import type { Scene3DDocument, Scene3DSlot } from './types.ts' + +const SLOT_COLORS: Record = { + subject_1: [40, 140, 220], + subject_2: [220, 90, 70], + background: [60, 60, 70], + prop: [200, 180, 60], +} + +export function scene3dSlotColor(slot: Scene3DSlot['slot']): [number, number, number] { + return SLOT_COLORS[slot] ?? [180, 180, 180] +} + +export function createDefaultScene3DDocument(): Scene3DDocument { + return { + version: 1, + units: 'meters', + up: 'y', + width: 1280, + height: 720, + fps: 30, + duration: 4, + camera: { + family: 'establishment', + eye: [0, 1.6, 4.2], + look: [0, 1, 0], + fov: 50, + orbitRadius: 4.2, + orbitHeight: 1.6, + orbitTurns: 1, + }, + light: { + kind: 'directional', + direction: [-0.35, -1, -0.25], + intensity: 1.15, + color: '#fff4e5', + }, + slots: [ + { + id: 'subject_1', + slot: 'subject_1', + position: [-0.85, 0, 0], + rotationY: 0.35, + scale: 1, + sourceUrl: '', + clip: null, + }, + { + id: 'subject_2', + slot: 'subject_2', + position: [0.85, 0, 0], + rotationY: -0.35, + scale: 1, + sourceUrl: '', + clip: null, + }, + ], + } +} + +export function cloneScene3DDocument(document: Scene3DDocument): Scene3DDocument { + return structuredClone(document) +} + +export function parseScene3DDocument(raw: unknown): Scene3DDocument | null { + if (!raw || typeof raw !== 'object') return null + const value = raw as Partial + if (value.version !== 1 || value.units !== 'meters' || value.up !== 'y') return null + if (!Array.isArray(value.slots) || !value.camera || !value.light) return null + return value as Scene3DDocument +} diff --git a/ui/src/features/scene3d/gpu.ts b/ui/src/features/scene3d/gpu.ts new file mode 100644 index 00000000..fd65ea97 --- /dev/null +++ b/ui/src/features/scene3d/gpu.ts @@ -0,0 +1,209 @@ +import { + AnimationMixer, + Box3, + BoxGeometry, + CircleGeometry, + Color, + DirectionalLight, + HemisphereLight, + Mesh, + MeshStandardMaterial, + Object3D, + PerspectiveCamera, + Scene, + Vector3, + WebGLRenderer, + type Material, + type Texture, +} from 'three' +import type { GLTF } from 'three/addons/loaders/GLTFLoader.js' +import { scene3dSlotColor } from './document.ts' +import type { Scene3DClipCatalogEntry, Scene3DLight, Scene3DSlot } from './types.ts' + +export const MAX_VIEW_WIDTH = 1280 +export const MAX_VIEW_HEIGHT = 720 +export const MAX_PIXEL_RATIO = 1.25 + +export type SlotGpu = { + sourceUrl: string + clipKey: string + root: Object3D + animations: GLTF['animations'] + mixer: AnimationMixer | null +} + +export type GpuWorld = { + renderer: WebGLRenderer + scene: Scene + camera: PerspectiveCamera + dir: DirectionalLight + slots: Map +} + +export function clipKeyOf(clip: Scene3DSlot['clip']): string { + return clip ? `${clip.index}\0${clip.name}` : '' +} + +export function catalogFromClips(animations: GLTF['animations']): Scene3DClipCatalogEntry[] { + return animations.map((clip, index) => ({ + index, + name: clip.name, + durationSeconds: Number.isFinite(clip.duration) && clip.duration > 0 ? clip.duration : null, + })) +} + +function isTexture(value: unknown): value is Texture { + return Boolean(value && typeof value === 'object' && 'isTexture' in value) +} + +export function disposeMaterial(material: Material) { + material.dispose() + for (const value of Object.values(material)) { + if (isTexture(value)) value.dispose() + } +} + +export function disposeObject(object: Object3D) { + object.traverse(child => { + if (!(child instanceof Mesh)) return + child.geometry.dispose() + const materials = Array.isArray(child.material) ? child.material : [child.material] + for (const material of materials) disposeMaterial(material) + }) +} + +export function viewSize(host: HTMLElement) { + const width = host.clientWidth || 640 + const height = host.clientHeight || 360 + const scale = Math.min(1, MAX_VIEW_WIDTH / width, MAX_VIEW_HEIGHT / height) + return { + width: Math.max(1, Math.round(width * scale)), + height: Math.max(1, Math.round(height * scale)), + } +} + +export function applyLight(dir: DirectionalLight, light: Scene3DLight) { + dir.color = new Color(light.color) + dir.intensity = light.intensity + dir.position + .set(-light.direction[0], -light.direction[1], -light.direction[2]) + .normalize() + .multiplyScalar(6) +} + +export function placeholderMesh(slot: Scene3DSlot) { + const color = scene3dSlotColor(slot.slot) + const mesh = new Mesh( + new BoxGeometry(0.6, 1.6, 0.6), + new MeshStandardMaterial({ color: new Color(color[0] / 255, color[1] / 255, color[2] / 255) }), + ) + mesh.position.set(slot.position[0], 0.8, slot.position[2]) + mesh.rotation.y = slot.rotationY + return mesh +} + +export function fitGltf(root: Object3D, slot: Scene3DSlot) { + const box = new Box3().setFromObject(root) + const size = new Vector3() + box.getSize(size) + root.scale.setScalar((1.7 * slot.scale) / Math.max(size.y, 0.001)) + root.position.set(slot.position[0], 0, slot.position[2]) + root.rotation.y = slot.rotationY +} + +export function bindMixer(root: Object3D, animations: GLTF['animations'], slot: Scene3DSlot) { + if (!slot.clip) return null + const clip = animations[slot.clip.index] + if (!clip || clip.name !== slot.clip.name) return null + const mixer = new AnimationMixer(root) + mixer.clipAction(clip).play() + return mixer +} + +export function dropSlot(world: GpuWorld, slotId: string) { + const current = world.slots.get(slotId) + if (!current) return + current.mixer?.stopAllAction() + world.scene.remove(current.root) + disposeObject(current.root) + world.slots.delete(slotId) +} + +export function placeSlot( + world: GpuWorld, + slot: Scene3DSlot, + root: Object3D, + animations: GLTF['animations'], +) { + dropSlot(world, slot.id) + world.scene.add(root) + world.slots.set(slot.id, { + sourceUrl: slot.sourceUrl, + clipKey: clipKeyOf(slot.clip), + root, + animations, + mixer: bindMixer(root, animations, slot), + }) +} + +export function syncSlotClip(world: GpuWorld, slot: Scene3DSlot) { + const current = world.slots.get(slot.id) + if (!current) return + const nextKey = clipKeyOf(slot.clip) + if (current.clipKey === nextKey) return + current.mixer?.stopAllAction() + current.mixer = bindMixer(current.root, current.animations, slot) + current.clipKey = nextKey +} + +export function pruneSlots(world: GpuWorld, slots: readonly Scene3DSlot[]) { + const wanted = new Set(slots.map(slot => slot.id)) + for (const id of [...world.slots.keys()]) { + if (!wanted.has(id)) dropSlot(world, id) + } +} + +export function createWorld(host: HTMLDivElement, light: Scene3DLight, fov: number): GpuWorld { + const renderer = new WebGLRenderer({ antialias: true, alpha: false, powerPreference: 'low-power' }) + renderer.setPixelRatio(Math.min(window.devicePixelRatio || 1, MAX_PIXEL_RATIO)) + renderer.shadowMap.enabled = false + renderer.domElement.style.display = 'block' + renderer.domElement.style.width = '100%' + renderer.domElement.style.height = '100%' + host.append(renderer.domElement) + const scene = new Scene() + scene.background = new Color(0x10141c) + const camera = new PerspectiveCamera(fov, 16 / 9, 0.05, 80) + scene.add(new HemisphereLight(0xc8d8ff, 0x2a2118, 0.55)) + const dir = new DirectionalLight(light.color, light.intensity) + applyLight(dir, light) + scene.add(dir) + const floor = new Mesh( + new CircleGeometry(8, 48), + new MeshStandardMaterial({ color: 0x1c222c, roughness: 0.92 }), + ) + floor.rotation.x = -Math.PI / 2 + scene.add(floor) + return { renderer, scene, camera, dir, slots: new Map() } +} + +export function disposeWorld(world: GpuWorld) { + for (const id of [...world.slots.keys()]) dropSlot(world, id) + disposeObject(world.scene) + world.renderer.dispose() + world.renderer.forceContextLoss() + world.renderer.domElement.remove() +} + +export function resizeWorld(world: GpuWorld, host: HTMLDivElement) { + const size = viewSize(host) + world.renderer.setSize(size.width, size.height, false) + world.camera.aspect = size.width / size.height + world.camera.updateProjectionMatrix() +} + +export function poseLoadedSlot(current: SlotGpu, slot: Scene3DSlot) { + if (!slot.sourceUrl) return + current.root.position.set(slot.position[0], 0, slot.position[2]) + current.root.rotation.y = slot.rotationY +} diff --git a/ui/src/features/scene3d/index.ts b/ui/src/features/scene3d/index.ts new file mode 100644 index 00000000..95a534de --- /dev/null +++ b/ui/src/features/scene3d/index.ts @@ -0,0 +1,17 @@ +export type { + Scene3DCamera, + Scene3DCameraFamily, + Scene3DClipCatalogEntry, + Scene3DClipError, + Scene3DClipRef, + Scene3DDocument, + Scene3DLight, + Scene3DSlot, + Scene3DSlotId, + Vec3, +} from './types.ts' +export { cameraEyeAtTime, orbitEye, projectPoint } from './camera.ts' +export { clipBindingError, resolveScene3DClip } from './clips.ts' +export { scene3dClipLocalTime, scene3dFrameCount, scene3dFrameTime } from './clock.ts' +export { cloneScene3DDocument, createDefaultScene3DDocument, parseScene3DDocument } from './document.ts' +export { hashSoftwareFrame, renderScene3DSoftware } from './softwareRender.ts' diff --git a/ui/src/features/scene3d/softwareRender.ts b/ui/src/features/scene3d/softwareRender.ts new file mode 100644 index 00000000..3ba8f359 --- /dev/null +++ b/ui/src/features/scene3d/softwareRender.ts @@ -0,0 +1,62 @@ +import { cameraEyeAtTime, projectPoint } from './camera.ts' +import { scene3dSlotColor } from './document.ts' +import type { Scene3DDocument } from './types.ts' + +export type SoftwareFrame = { + width: number + height: number + pixels: Uint8Array +} + +function fillRect( + frame: SoftwareFrame, + x0: number, + y0: number, + x1: number, + y1: number, + rgb: [number, number, number], +) { + const left = Math.max(0, Math.floor(Math.min(x0, x1))) + const right = Math.min(frame.width - 1, Math.ceil(Math.max(x0, x1))) + const top = Math.max(0, Math.floor(Math.min(y0, y1))) + const bottom = Math.min(frame.height - 1, Math.ceil(Math.max(y0, y1))) + for (let y = top; y <= bottom; y += 1) { + for (let x = left; x <= right; x += 1) { + const i = (y * frame.width + x) * 4 + frame.pixels[i] = rgb[0] + frame.pixels[i + 1] = rgb[1] + frame.pixels[i + 2] = rgb[2] + frame.pixels[i + 3] = 255 + } + } +} + +export function renderScene3DSoftware(document: Scene3DDocument, sceneSeconds: number): SoftwareFrame { + const width = 160 + const height = Math.max(1, Math.round(160 * document.height / Math.max(1, document.width))) + const pixels = new Uint8Array(width * height * 4) + pixels.fill(18) + for (let i = 3; i < pixels.length; i += 4) pixels[i] = 255 + const frame = { width, height, pixels } + fillRect(frame, 0, height * 0.62, width, height, [32, 34, 38]) + const eye = cameraEyeAtTime(document.camera, sceneSeconds, document.duration) + const aspect = width / height + for (const slot of document.slots) { + const projected = projectPoint(slot.position, eye, document.camera.look, document.camera.fov, aspect) + if (!projected) continue + const size = Math.max(6, 28 * slot.scale / Math.max(0.4, projected.depth)) + const cx = projected.x * width + const cy = projected.y * height + fillRect(frame, cx - size, cy - size * 1.6, cx + size, cy + size * 0.4, scene3dSlotColor(slot.slot)) + } + return frame +} + +export function hashSoftwareFrame(frame: SoftwareFrame): string { + let hash = 2166136261 + for (let i = 0; i < frame.pixels.length; i += 1) { + hash ^= frame.pixels[i] + hash = Math.imul(hash, 16777619) + } + return (hash >>> 0).toString(16).padStart(8, '0') +} diff --git a/ui/src/features/scene3d/types.ts b/ui/src/features/scene3d/types.ts new file mode 100644 index 00000000..68d3863c --- /dev/null +++ b/ui/src/features/scene3d/types.ts @@ -0,0 +1,69 @@ +export type Vec3 = readonly [number, number, number] + +export type Scene3DCameraFamily = + | 'establishment' + | 'follow' + | 'orbit' + | 'reveal' + | 'encounter' + | 'pursuit' + | 'product' + | 'musical' + +export type Scene3DSlotId = 'subject_1' | 'subject_2' | 'background' | 'prop' + +export type Scene3DClipRef = { + index: number + name: string +} + +export type Scene3DSlot = { + id: string + slot: Scene3DSlotId + position: Vec3 + rotationY: number + scale: number + sourceUrl: string + clip: Scene3DClipRef | null +} + +export type Scene3DCamera = { + family: Scene3DCameraFamily + eye: Vec3 + look: Vec3 + fov: number + orbitRadius?: number + orbitHeight?: number + orbitTurns?: number +} + +export type Scene3DLight = { + kind: 'directional' + direction: Vec3 + intensity: number + color: string +} + +export type Scene3DDocument = { + version: 1 + units: 'meters' + up: 'y' + width: number + height: number + fps: 24 | 30 | 60 + duration: number + camera: Scene3DCamera + light: Scene3DLight + slots: Scene3DSlot[] +} + +export type Scene3DClipCatalogEntry = { + index: number + name: string + durationSeconds: number | null +} + +export type Scene3DClipError = { + code: 'clip_missing' | 'clip_name_mismatch' + message: string +} diff --git a/ui/src/i18n/locales/en/scene3d.json b/ui/src/i18n/locales/en/scene3d.json index c628b573..b3f12a14 100644 --- a/ui/src/i18n/locales/en/scene3d.json +++ b/ui/src/i18n/locales/en/scene3d.json @@ -885,5 +885,30 @@ "cameraN": "Camera {{n}}", "eventN": "Event {{n}}", "photoCamera": "Photo camera · {{label}}" + }, + "stage": { + "badge": "3D stage", + "compositor": "2.5D compositor", + "world": "3D stage", + "play": "Play", + "pause": "Pause", + "help": "Metres, Y-up. Clip identity is index plus the exact glTF name. Local GLB files stay in the browser; they are not uploaded.", + "clipIndex": "Clip index", + "clipName": "Exact name", + "noClip": "No clip", + "exactName": "Running", + "eye": "eye x {{x}}", + "slot": { + "subject_1": "Subject 1", + "subject_2": "Subject 2", + "background": "Background", + "prop": "Prop" + }, + "family": { + "establishment": "Establishment", + "orbit": "Orbit", + "follow": "Follow", + "product": "Product" + } } } diff --git a/ui/src/i18n/locales/es/scene3d.json b/ui/src/i18n/locales/es/scene3d.json index 42e469d8..b3ea789a 100644 --- a/ui/src/i18n/locales/es/scene3d.json +++ b/ui/src/i18n/locales/es/scene3d.json @@ -885,5 +885,30 @@ "cameraN": "Cámara {{n}}", "eventN": "Evento {{n}}", "photoCamera": "Cámara fotográfica · {{label}}" + }, + "stage": { + "badge": "Escenario 3D", + "compositor": "Compositor 2,5D", + "world": "Escenario 3D", + "play": "Reproducir", + "pause": "Pausa", + "help": "Metros, Y-up. La identidad del clip es índice más el nombre glTF exacto. Los GLB locales se quedan en el navegador; no se suben.", + "clipIndex": "Índice de clip", + "clipName": "Nombre exacto", + "noClip": "Sin clip", + "exactName": "Running", + "eye": "ojo x {{x}}", + "slot": { + "subject_1": "Sujeto 1", + "subject_2": "Sujeto 2", + "background": "Fondo", + "prop": "Atrezo" + }, + "family": { + "establishment": "Establecimiento", + "orbit": "Órbita", + "follow": "Seguimiento", + "product": "Producto" + } } } diff --git a/ui/tests/scene3dStage.test.mjs b/ui/tests/scene3dStage.test.mjs new file mode 100644 index 00000000..eb87d4da --- /dev/null +++ b/ui/tests/scene3dStage.test.mjs @@ -0,0 +1,71 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { cameraEyeAtTime, orbitEye } from '../src/features/scene3d/camera.ts' +import { clipBindingError, resolveScene3DClip } from '../src/features/scene3d/clips.ts' +import { scene3dClipLocalTime, scene3dFrameCount, scene3dFrameTime } from '../src/features/scene3d/clock.ts' +import { cloneScene3DDocument, createDefaultScene3DDocument, parseScene3DDocument } from '../src/features/scene3d/document.ts' +import { hashSoftwareFrame, renderScene3DSoftware } from '../src/features/scene3d/softwareRender.ts' + +test('frame clock matches compositor export indexing', () => { + assert.equal(scene3dFrameCount(2, 30), 60) + assert.equal(scene3dFrameTime(0, 2, 30), 0) + assert.equal(scene3dFrameTime(59, 2, 30), 59 / 30) + assert.ok(scene3dFrameTime(59, 2, 30) < 2) +}) + +test('unknown clip duration is not treated as zero playback', () => { + assert.equal(scene3dClipLocalTime(1.2, null), null) + assert.equal(scene3dClipLocalTime(1.2, 0), null) + assert.equal(scene3dClipLocalTime(0.5, 1, { loop: true }), 0.5) +}) + +test('clip identity is index plus exact name', () => { + const catalog = [ + { index: 0, name: 'Armature|clip0|baselayer', durationSeconds: 0.033 }, + { index: 1, name: 'Running', durationSeconds: 0.667 }, + ] + const running = resolveScene3DClip(catalog, { index: 1, name: 'Running' }) + assert.equal(running && 'name' in running && running.name, 'Running') + const missing = clipBindingError(resolveScene3DClip(catalog, { index: 9, name: 'Running' })) + assert.equal(missing?.code, 'clip_missing') + const mismatch = clipBindingError(resolveScene3DClip(catalog, { index: 1, name: 'run' })) + assert.equal(mismatch?.code, 'clip_name_mismatch') +}) + +test('clip catalogs are per slot, not a shared name list', () => { + const subject = [ + { index: 0, name: 'Armature|clip0|baselayer', durationSeconds: 0.033 }, + { index: 1, name: 'Running', durationSeconds: 0.667 }, + ] + const prop = [{ index: 0, name: 'Spin', durationSeconds: 2 }] + assert.equal(clipBindingError(resolveScene3DClip(subject, { index: 1, name: 'Running' })), null) + assert.equal(clipBindingError(resolveScene3DClip(prop, { index: 1, name: 'Running' }))?.code, 'clip_missing') +}) + +test('save and reopen keeps camera, light, slots and clip refs', () => { + const original = createDefaultScene3DDocument() + original.slots[0].clip = { index: 1, name: 'Running' } + original.camera.family = 'orbit' + const restored = parseScene3DDocument(JSON.parse(JSON.stringify(cloneScene3DDocument(original)))) + assert.ok(restored) + assert.equal(restored.camera.family, 'orbit') + assert.deepEqual(restored.slots[0].clip, { index: 1, name: 'Running' }) + assert.equal(restored.light.kind, 'directional') + assert.equal(restored.slots.length, 2) +}) + +test('software frames differ when subjects move or the camera orbits', () => { + const document = createDefaultScene3DDocument() + const first = hashSoftwareFrame(renderScene3DSoftware(document, 0)) + document.slots[0].position = [1.4, 0, 0] + const moved = hashSoftwareFrame(renderScene3DSoftware(document, 0)) + assert.notEqual(first, moved) + const orbitDoc = createDefaultScene3DDocument() + orbitDoc.camera.family = 'orbit' + const a = hashSoftwareFrame(renderScene3DSoftware(orbitDoc, 0)) + const b = hashSoftwareFrame(renderScene3DSoftware(orbitDoc, 2)) + assert.notEqual(a, b) + const eye0 = cameraEyeAtTime(orbitDoc.camera, 0, 4) + const eyeHalf = orbitEye(orbitDoc.camera.look, 4.2, 1.6, Math.PI) + assert.notEqual(eye0[0], eyeHalf[0]) +}) From fe35f439f2cea53ce67abe13054c27d87f85c49c Mon Sep 17 00:00:00 2001 From: IAnMove <216241348+IAnMove@users.noreply.github.com> Date: Mon, 7 Sep 2026 00:19:11 +0200 Subject: [PATCH 2/7] feat(video3d): pick assets from a blocking explorer dialog Replace filename-only resource dropdowns in 3D Video with a modal grid (preview, name, creation date, choose/cancel), matching the scene library overlay. The 2.5D | 3D stage switch sits at the top of the editor so the mode is visible without hunting 9px buttons. --- .../components/Sidebar/SceneAnimatorPanel.tsx | 81 +++++-- .../components/Sidebar/SceneRecipePanel.tsx | 66 +++--- .../components/common/AssetExplorerDialog.tsx | 222 ++++++++++++++++++ ui/src/components/common/assetExplorer.ts | 13 + ui/src/features/scene3d/Scene3DWorkspace.tsx | 42 +++- ui/src/i18n/locales/en/common.json | 20 ++ ui/src/i18n/locales/en/scene3d.json | 3 + ui/src/i18n/locales/es/common.json | 20 ++ ui/src/i18n/locales/es/scene3d.json | 3 + ui/tests/assetExplorerDialog.test.tsx | 91 +++++++ 10 files changed, 496 insertions(+), 65 deletions(-) create mode 100644 ui/src/components/common/AssetExplorerDialog.tsx create mode 100644 ui/src/components/common/assetExplorer.ts create mode 100644 ui/tests/assetExplorerDialog.test.tsx diff --git a/ui/src/components/Sidebar/SceneAnimatorPanel.tsx b/ui/src/components/Sidebar/SceneAnimatorPanel.tsx index 822b3ef9..9f6979a8 100644 --- a/ui/src/components/Sidebar/SceneAnimatorPanel.tsx +++ b/ui/src/components/Sidebar/SceneAnimatorPanel.tsx @@ -1,10 +1,11 @@ import { memo, useCallback, useEffect, useRef, useState, type CSSProperties, type PointerEvent as ReactPointerEvent } from 'react' import type { ParseKeys } from 'i18next' -import { AlignHorizontalJustifyCenter, AlignVerticalJustifyCenter, Box, Camera, ChevronDown, ChevronDown as Down, ChevronUp, CloudRain, Copy, CopyPlus, Download, Eye, EyeOff, FileJson, Film, FolderOpen, Grid3X3, Image as ImageIcon, Loader2, Lock, Magnet, Mic, Play, Plus, Redo2, Save, Trash2, Undo2, Unlock, Video } from 'lucide-react' +import { AlignHorizontalJustifyCenter, AlignVerticalJustifyCenter, Box, Camera, ChevronDown, ChevronUp, CloudRain, Copy, CopyPlus, Download, Eye, EyeOff, FileJson, Film, FolderOpen, Grid3X3, Image as ImageIcon, Loader2, Lock, Magnet, Mic, Play, Plus, Redo2, Save, Trash2, Undo2, Unlock, Video } from 'lucide-react' import { ArrayBufferTarget, Muxer } from 'mp4-muxer' import { useUiTranslation } from '../../i18n' import { useStore } from '../../stores/useStore' -import { analyzeAudio, deleteCharacterKit, fetchCharacterKitLibrary, fetchOutputs, generateLlmText, saveCharacterKit, saveScene as saveSceneOutput, saveSceneRecording, uploadImage } from '../../api/client' +import { analyzeAudio, deleteCharacterKit, fetchCharacterKitLibrary, fetchOutputs, generateLlmText, saveCharacterKit, saveScene as saveSceneOutput, saveSceneRecording, uploadImage, type ApiOutput } from '../../api/client' +import { AssetExplorerDialog, AssetPickTrigger } from '../common/AssetExplorerDialog' import { generateSceneSpeechClip } from '../../lib/sceneSpeech' import { SceneRecipePanel } from './SceneRecipePanel' import { TemplateComposerDialog } from '../../features/sceneTemplates/TemplateComposerDialog' @@ -38,6 +39,20 @@ type Point = { x: number; y: number; scale: number; opacity?: number; rotation?: type AnimatorLayerType = SceneLayerType type VisualLayerType = Exclude type ParallaxPreset = 'background' | 'midground' | 'foreground' +type AssetExplorerPurpose = 'layer-model' | 'layer-media' | 'narrative-hero' | 'narrative-plate' | 'narrative-prop' | 'narrative-foreground' | 'scene-audio' + +function assetsForExplorer( + purpose: AssetExplorerPurpose, + models: ApiOutput[], + media: ApiOutput[], + visuals: ApiOutput[], + audio: ApiOutput[], +): ApiOutput[] { + if (purpose === 'layer-model') return models + if (purpose === 'layer-media' || purpose === 'narrative-plate' || purpose === 'narrative-foreground') return media + if (purpose === 'scene-audio') return audio + return visuals +} type AnimatorLayer = Omit & { type: AnimatorLayerType /** Camera-pan response. Distant layers move less; foreground layers move more. */ @@ -487,7 +502,7 @@ export function SceneAnimatorPanel() { const [selectedId, setSelectedId] = useState(null) const [addOpen, setAddOpen] = useState(false) const [templateComposerOpen, setTemplateComposerOpen] = useState(false) - const [picker, setPicker] = useState<'model' | 'media' | null>(null) + const [assetExplorer, setAssetExplorer] = useState(null) const [playing, setPlaying] = useState(false) const [recording, setRecording] = useState(false) const [publishing, setPublishing] = useState(false) @@ -903,7 +918,7 @@ export function SceneAnimatorPanel() { const layers = type === 'image' || type === 'video' ? [layer, ...ordered] : [...ordered, layer] return { ...current, layers: normalizeZ(layers) } }) - setSelectedId(id); setAddOpen(false); setPicker(null) + setSelectedId(id); setAddOpen(false); setAssetExplorer(null) } const addAtmosphere = (kind: SceneAtmosphereKind) => { const id = uid() @@ -932,7 +947,7 @@ export function SceneAnimatorPanel() { } return { ...current, layers: normalizeZ([...current.layers, layer]) } }) - setSelectedId(id); setAddOpen(false); setPicker(null) + setSelectedId(id); setAddOpen(false); setAssetExplorer(null) } const addCamera = () => { const id = uid() @@ -956,7 +971,7 @@ export function SceneAnimatorPanel() { const layers = current.layers.map(layer => layer.type === 'camera' ? { ...layer, visible: false } : layer) return { ...current, layers: normalizeZ([...layers, camera]) } }) - setSelectedId(id); setAddOpen(false); setPicker(null); setProgress(0) + setSelectedId(id); setAddOpen(false); setAssetExplorer(null); setProgress(0) } const duplicateLayer = (id: string) => { const original = sceneRef.current.layers.find(layer => layer.id === id) @@ -2932,6 +2947,19 @@ export function SceneAnimatorPanel() {
updateScene(current => ({ ...current, name: event.target.value }))} aria-label={t('animator.sceneNameAria')} className="w-44 rounded border border-transparent bg-transparent px-1 py-0.5 text-xs font-medium hover:border-border focus:border-accent-blue focus:outline-none" />{scene.width}×{scene.height}
{lastAutosaveAt ? t('animator.autosaved', { time: new Date(lastAutosaveAt).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }) }) : t('animator.autosaveWaiting')}
+
+
{t(scene3dKey('stage.modeTitle'))}
+

{t(scene3dKey('stage.modeHelp'))}

+
+ + +
+
{RESOLUTIONS.map(([label, width, height]) => )}{t('animator.frameRate')}{([30, 60] as SceneFrameRate[]).map(rate => )}
@@ -2942,15 +2970,6 @@ export function SceneAnimatorPanel() {
{selected && isVisualLayer(selected) && selected.type !== 'model3d' && selected.type !== 'effect' && } {selected && isVisualLayer(selected) && selected.type !== 'model3d' && selected.type !== 'effect' && } -
- - -
{stageMode === 'world3d' ?
:
{[...scene.layers].sort((a, b) => a.z - b.z).map(renderLayer)} @@ -2996,13 +3015,13 @@ export function SceneAnimatorPanel() {
{NARRATIVE_SCENE_TEMPLATES.map(template => )}

{narrativeTemplate.description}

- + asset.name === narrativeHero)} placeholder={t('animator.chooseAsset')} disabled={playing || recording || publishing} onOpen={() => setAssetExplorer('narrative-hero')} /> {narrativeHero &&

{narrativeSuitability('hero', narrativeHero).message}

} - + asset.name === narrativePlate)} placeholder={t('animator.chooseAsset')} disabled={playing || recording || publishing} onOpen={() => setAssetExplorer('narrative-plate')} /> {narrativePlate && narrativeSuitability('plate', narrativePlate).level !== 'ok' &&

{narrativeSuitability('plate', narrativePlate).message}

} {narrativePlate && } - {narrativeTemplate.assetSlots.some(slot => slot.id === 'prop') && } - {narrativeTemplate.assetSlots.some(slot => slot.id === 'foreground') && } + {narrativeTemplate.assetSlots.some(slot => slot.id === 'prop') && slot.id === 'prop')?.required ? '' : t('animator.optional')}`} selected={narrativeVisuals.find(asset => asset.name === narrativeProp)} placeholder={t('animator.none')} disabled={playing || recording || publishing} onOpen={() => setAssetExplorer('narrative-prop')} />} + {narrativeTemplate.assetSlots.some(slot => slot.id === 'foreground') && asset.name === narrativeForeground)} placeholder={t('animator.none')} disabled={playing || recording || publishing} onOpen={() => setAssetExplorer('narrative-foreground')} />}
{narrativeTemplate.controls.includes('mood') && } {narrativeTemplate.controls.includes('intensity') && } @@ -3026,7 +3045,7 @@ export function SceneAnimatorPanel() {

{t('animator.sceneAudioHelp')}