diff --git a/public/audio/upbeat-happy-corporate.mp3 b/public/audio/upbeat-happy-corporate.mp3 new file mode 100644 index 00000000..6a36726c Binary files /dev/null and b/public/audio/upbeat-happy-corporate.mp3 differ diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index 4e0c0676..71e99aab 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -7,6 +7,7 @@ import { FolderOpen, Image, Lock, + Music, Palette, Save, Sparkles, @@ -36,6 +37,7 @@ import { type AspectRatio } from "@/utils/aspectRatioUtils"; import { getTestId } from "@/utils/getTestId"; import { AnnotationSettingsPanel } from "./AnnotationSettingsPanel"; import { CropControl } from "./CropControl"; +import { AUDIO_TRACKS, DEFAULT_BG_MUSIC_VOLUME, GRADIENTS, WALLPAPERS } from "./constants"; import { KeyboardShortcutsHelp } from "./KeyboardShortcutsHelp"; import type { AnnotationRegion, @@ -47,38 +49,6 @@ import type { } from "./types"; import { SPEED_OPTIONS } from "./types"; -const WALLPAPER_COUNT = 18; -const WALLPAPER_RELATIVE = Array.from( - { length: WALLPAPER_COUNT }, - (_, i) => `wallpapers/wallpaper${i + 1}.jpg`, -); -const GRADIENTS = [ - "linear-gradient( 111.6deg, rgba(114,167,232,1) 9.4%, rgba(253,129,82,1) 43.9%, rgba(253,129,82,1) 54.8%, rgba(249,202,86,1) 86.3% )", - "linear-gradient(120deg, #d4fc79 0%, #96e6a1 100%)", - "radial-gradient( circle farthest-corner at 3.2% 49.6%, rgba(80,12,139,0.87) 0%, rgba(161,10,144,0.72) 83.6% )", - "linear-gradient( 111.6deg, rgba(0,56,68,1) 0%, rgba(163,217,185,1) 51.5%, rgba(231, 148, 6, 1) 88.6% )", - "linear-gradient( 107.7deg, rgba(235,230,44,0.55) 8.4%, rgba(252,152,15,1) 90.3% )", - "linear-gradient( 91deg, rgba(72,154,78,1) 5.2%, rgba(251,206,70,1) 95.9% )", - "radial-gradient( circle farthest-corner at 10% 20%, rgba(2,37,78,1) 0%, rgba(4,56,126,1) 19.7%, rgba(85,245,221,1) 100.2% )", - "linear-gradient( 109.6deg, rgba(15,2,2,1) 11.2%, rgba(36,163,190,1) 91.1% )", - "linear-gradient(135deg, #FBC8B4, #2447B1)", - "linear-gradient(109.6deg, #F635A6, #36D860)", - "linear-gradient(90deg, #FF0101, #4DFF01)", - "linear-gradient(315deg, #EC0101, #5044A9)", - "linear-gradient(45deg, #ff9a9e 0%, #fad0c4 99%, #fad0c4 100%)", - "linear-gradient(to top, #a18cd1 0%, #fbc2eb 100%)", - "linear-gradient(to right, #ff8177 0%, #ff867a 0%, #ff8c7f 21%, #f99185 52%, #cf556c 78%, #b12a5b 100%)", - "linear-gradient(120deg, #84fab0 0%, #8fd3f4 100%)", - "linear-gradient(to right, #4facfe 0%, #00f2fe 100%)", - "linear-gradient(to top, #fcc5e4 0%, #fda34b 15%, #ff7882 35%, #c8699e 52%, #7046aa 71%, #0c1db8 87%, #020f75 100%)", - "linear-gradient(to right, #fa709a 0%, #fee140 100%)", - "linear-gradient(to top, #30cfd0 0%, #330867 100%)", - "linear-gradient(to top, #c471f5 0%, #fa71cd 100%)", - "linear-gradient(to right, #f78ca0 0%, #f9748f 19%, #fd868c 60%, #fe9a8b 100%)", - "linear-gradient(to top, #48c6ef 0%, #6f86d6 100%)", - "linear-gradient(to right, #0acffe 0%, #495aff 100%)", -]; - interface SettingsPanelProps { selected: string; onWallpaperChange: (path: string) => void; @@ -132,6 +102,10 @@ interface SettingsPanelProps { selectedSpeedValue?: PlaybackSpeed | null; onSpeedChange?: (speed: PlaybackSpeed) => void; onSpeedDelete?: (id: string) => void; + backgroundMusic?: string | null; + onBackgroundMusicChange?: (trackId: string | null) => void; + backgroundMusicVolume?: number; + onBackgroundMusicVolumeChange?: (volume: number) => void; } export default SettingsPanel; @@ -197,6 +171,10 @@ export function SettingsPanel({ selectedSpeedValue, onSpeedChange, onSpeedDelete, + backgroundMusic, + onBackgroundMusicChange, + backgroundMusicVolume = DEFAULT_BG_MUSIC_VOLUME, + onBackgroundMusicVolumeChange, }: SettingsPanelProps) { const [wallpaperPaths, setWallpaperPaths] = useState([]); const [customImages, setCustomImages] = useState([]); @@ -206,10 +184,10 @@ export function SettingsPanel({ let mounted = true; (async () => { try { - const resolved = await Promise.all(WALLPAPER_RELATIVE.map((p) => getAssetPath(p))); + const resolved = await Promise.all(WALLPAPERS.map((w) => getAssetPath(w.src))); if (mounted) setWallpaperPaths(resolved); - } catch (_err) { - if (mounted) setWallpaperPaths(WALLPAPER_RELATIVE.map((p) => `/${p}`)); + } catch { + if (mounted) setWallpaperPaths(WALLPAPERS.map((w) => `/${w.src}`)); } })(); return () => { @@ -236,7 +214,11 @@ export function SettingsPanel({ ]; const [selectedColor, setSelectedColor] = useState("#ADADAD"); - const [gradient, setGradient] = useState(GRADIENTS[0]); + const [customAudioTracks, setCustomAudioTracks] = useState< + Array<{ label: string; dataUrl: string }> + >([]); + const audioFileInputRef = useRef(null); + const [gradient, setGradient] = useState(GRADIENTS[0].value); const [showCropModal, setShowCropModal] = useState(false); const cropSnapshotRef = useRef(null); const [cropAspectLocked, setCropAspectLocked] = useState(false); @@ -397,7 +379,7 @@ export function SettingsPanel({ setCustomImages((prev) => prev.filter((img) => img !== imageUrl)); // If the removed image was selected, clear selection if (selected === imageUrl) { - onWallpaperChange(wallpaperPaths[0] || WALLPAPER_RELATIVE[0]); + onWallpaperChange(wallpaperPaths[0] || WALLPAPERS[0].src); } }; @@ -415,6 +397,36 @@ export function SettingsPanel({ setShowCropModal(false); }; + const handleAudioUpload = (event: React.ChangeEvent) => { + const files = event.target.files; + if (!files || files.length === 0) return; + const file = files[0]; + if (file.type !== "audio/mpeg") { + toast.error("Invalid file type", { + description: "Please upload an MP3 audio file.", + }); + event.target.value = ""; + return; + } + const reader = new FileReader(); + reader.onload = (e) => { + const dataUrl = e.target?.result as string; + if (dataUrl) { + const label = file.name.replace(/\.mp3$/i, ""); + setCustomAudioTracks((prev) => [...prev, { label, dataUrl }]); + onBackgroundMusicChange?.(dataUrl); + toast.success("Custom audio uploaded!"); + } + }; + reader.onerror = () => { + toast.error("Failed to upload audio", { + description: "There was an error reading the file.", + }); + }; + reader.readAsDataURL(file); + event.target.value = ""; + }; + // Find selected annotation const selectedAnnotation = selectedAnnotationId ? annotationRegions.find((a) => a.id === selectedAnnotationId) @@ -444,6 +456,40 @@ export function SettingsPanel({ ); } + const renderTrackButton = ( + key: string, + label: string, + isActive: boolean, + onClick: () => void, + onDelete?: () => void, + ) => ( +
+ + )} + +
+ ); + return (
@@ -567,7 +613,11 @@ export function SettingsPanel({ )}
- +
@@ -748,7 +798,7 @@ export function SettingsPanel({ {(wallpaperPaths.length > 0 ? wallpaperPaths - : WALLPAPER_RELATIVE.map((p) => `/${p}`) + : WALLPAPERS.map((w) => `/${w.src}`) ).map((path) => { const isSelected = (() => { if (!selected) return false; @@ -806,18 +856,18 @@ export function SettingsPanel({
{GRADIENTS.map((g, idx) => (
{ - setGradient(g); - onWallpaperChange(g); + setGradient(g.value); + onWallpaperChange(g.value); }} role="button" /> @@ -828,6 +878,79 @@ export function SettingsPanel({ + + + +
+ + Sound Effects +
+ + +

Background Music

+
+ {AUDIO_TRACKS.map((track) => { + const isActive = backgroundMusic === track.id; + return renderTrackButton(track.id, track.label, isActive, () => + onBackgroundMusicChange?.(isActive ? null : track.id), + ); + })} + {customAudioTracks.map((track) => { + const isActive = backgroundMusic === track.dataUrl; + return renderTrackButton( + track.dataUrl, + track.label, + isActive, + () => onBackgroundMusicChange?.(isActive ? null : track.dataUrl), + () => { + setCustomAudioTracks((prev) => + prev.filter((t) => t.dataUrl !== track.dataUrl), + ); + if (isActive) { + onBackgroundMusicChange?.(null); + } + }, + ); + })} +
+ + + {backgroundMusic && ( +
+
+ Volume + + {Math.round(backgroundMusicVolume * 100)}% + +
+ onBackgroundMusicVolumeChange?.(values[0] / 100)} + min={0} + max={100} + step={1} + className="w-full" + /> +
+ )} + +
diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index cbf9b29b..26afc102 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -5,6 +5,7 @@ import { toast } from "sonner"; import { Toaster } from "@/components/ui/sonner"; import { useShortcuts } from "@/contexts/ShortcutsContext"; import { INITIAL_EDITOR_STATE, useEditorHistory } from "@/hooks/useEditorHistory"; +import { getAssetPath } from "@/lib/assetPath"; import { calculateOutputDimensions, type ExportFormat, @@ -20,6 +21,7 @@ import { import type { ProjectMedia } from "@/lib/recordingSession"; import { matchesShortcut } from "@/lib/shortcuts"; import { getAspectRatioValue, getNativeAspectRatioValue } from "@/utils/aspectRatioUtils"; +import { DEFAULT_BG_MUSIC_VOLUME, resolveBackgroundMusicUrl } from "./constants"; import { ExportDialog } from "./ExportDialog"; import PlaybackControls from "./PlaybackControls"; import { @@ -104,6 +106,8 @@ export default function VideoEditor() { const [gifLoop, setGifLoop] = useState(true); const [gifSizePreset, setGifSizePreset] = useState("medium"); const [exportedFilePath, setExportedFilePath] = useState(null); + const [backgroundMusic, setBackgroundMusic] = useState(null); + const [backgroundMusicVolume, setBackgroundMusicVolume] = useState(DEFAULT_BG_MUSIC_VOLUME); const [lastSavedSnapshot, setLastSavedSnapshot] = useState(null); const videoPlaybackRef = useRef(null); @@ -179,6 +183,8 @@ export default function VideoEditor() { setGifFrameRate(normalizedEditor.gifFrameRate); setGifLoop(normalizedEditor.gifLoop); setGifSizePreset(normalizedEditor.gifSizePreset); + setBackgroundMusic(normalizedEditor.backgroundMusic); + setBackgroundMusicVolume(normalizedEditor.backgroundMusicVolume); setSelectedZoomId(null); setSelectedTrimId(null); @@ -245,6 +251,8 @@ export default function VideoEditor() { gifFrameRate, gifLoop, gifSizePreset, + backgroundMusic, + backgroundMusicVolume, }), ); }, [ @@ -266,6 +274,8 @@ export default function VideoEditor() { gifFrameRate, gifLoop, gifSizePreset, + backgroundMusic, + backgroundMusicVolume, ]); const hasUnsavedChanges = Boolean( @@ -357,6 +367,8 @@ export default function VideoEditor() { gifFrameRate, gifLoop, gifSizePreset, + backgroundMusic, + backgroundMusicVolume, }); const fileNameBase = @@ -409,6 +421,8 @@ export default function VideoEditor() { gifFrameRate, gifLoop, gifSizePreset, + backgroundMusic, + backgroundMusicVolume, videoPath, ], ); @@ -1128,6 +1142,10 @@ export default function VideoEditor() { } } + const backgroundMusicUrl = backgroundMusic + ? await resolveBackgroundMusicUrl(backgroundMusic, getAssetPath) + : undefined; + const exporter = new VideoExporter({ videoUrl: videoPath, webcamVideoUrl: webcamVideoPath || undefined, @@ -1150,6 +1168,8 @@ export default function VideoEditor() { annotationRegions, previewWidth, previewHeight, + backgroundMusicUrl, + backgroundMusicVolume, onProgress: (progress: ExportProgress) => { setExportProgress(progress); }, @@ -1210,6 +1230,8 @@ export default function VideoEditor() { padding, cropRegion, annotationRegions, + backgroundMusic, + backgroundMusicVolume, isPlaying, aspectRatio, exportQuality, @@ -1377,6 +1399,8 @@ export default function VideoEditor() { onSelectAnnotation={handleSelectAnnotation} onAnnotationPositionChange={handleAnnotationPositionChange} onAnnotationSizeChange={handleAnnotationSizeChange} + backgroundMusic={backgroundMusic} + backgroundMusicVolume={backgroundMusicVolume} />
@@ -1516,6 +1540,10 @@ export default function VideoEditor() { } onSpeedChange={handleSpeedChange} onSpeedDelete={handleSpeedDelete} + backgroundMusic={backgroundMusic} + onBackgroundMusicChange={setBackgroundMusic} + backgroundMusicVolume={backgroundMusicVolume} + onBackgroundMusicVolumeChange={setBackgroundMusicVolume} /> diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx index 85029fcc..7183ca6b 100644 --- a/src/components/video-editor/VideoPlayback.tsx +++ b/src/components/video-editor/VideoPlayback.tsx @@ -26,6 +26,7 @@ import { getNativeAspectRatioValue, } from "@/utils/aspectRatioUtils"; import { AnnotationOverlay } from "./AnnotationOverlay"; +import { DEFAULT_BG_MUSIC_VOLUME, resolveBackgroundMusicUrl } from "./constants"; import { type AnnotationRegion, type SpeedRegion, @@ -84,6 +85,8 @@ interface VideoPlaybackProps { onSelectAnnotation?: (id: string | null) => void; onAnnotationPositionChange?: (id: string, position: { x: number; y: number }) => void; onAnnotationSizeChange?: (id: string, size: { width: number; height: number }) => void; + backgroundMusic?: string | null; + backgroundMusicVolume?: number; } export interface VideoPlaybackRef { @@ -128,6 +131,8 @@ const VideoPlayback = forwardRef( onSelectAnnotation, onAnnotationPositionChange, onAnnotationSizeChange, + backgroundMusic, + backgroundMusicVolume = DEFAULT_BG_MUSIC_VOLUME, }, ref, ) => { @@ -183,6 +188,83 @@ const VideoPlayback = forwardRef( const onTimeUpdateRef = useRef(onTimeUpdate); const onPlayStateChangeRef = useRef(onPlayStateChange); const videoReadyRafRef = useRef(null); + const bgAudioRef = useRef(null); + + // Background music management + // biome-ignore lint/correctness/useExhaustiveDependencies: backgroundMusicVolume is excluded — volume changes are handled by a dedicated effect + useEffect(() => { + if (bgAudioRef.current) { + bgAudioRef.current.pause(); + bgAudioRef.current = null; + } + + if (!backgroundMusic) return; + + let cancelled = false; + + const setupAudio = (audioUrl: string) => { + if (cancelled) return; + const audio = new Audio(audioUrl); + audio.loop = true; + audio.volume = backgroundMusicVolume; + audio.preload = "auto"; + bgAudioRef.current = audio; + + // Wait for audio to be ready, then auto-play if video is playing + const onReady = () => { + if (cancelled) return; + if (isPlayingRef.current) { + const video = videoRef.current; + if (video && audio.duration) { + audio.currentTime = video.currentTime % audio.duration; + } + audio.play().catch((_e) => { + /* autoplay blocked */ + }); + } + }; + audio.addEventListener("canplaythrough", onReady, { once: true }); + // Fire immediately if already ready (e.g. browser cache) + if (audio.readyState >= HTMLMediaElement.HAVE_ENOUGH_DATA) { + onReady(); + } + }; + + resolveBackgroundMusicUrl(backgroundMusic, getAssetPath).then(setupAudio); + + return () => { + cancelled = true; + if (bgAudioRef.current) { + bgAudioRef.current.pause(); + bgAudioRef.current = null; + } + }; + }, [backgroundMusic]); + + // Update background music volume + useEffect(() => { + if (bgAudioRef.current) { + bgAudioRef.current.volume = backgroundMusicVolume; + } + }, [backgroundMusicVolume]); + + // Sync background music with video play/pause + useEffect(() => { + const audio = bgAudioRef.current; + if (!audio || audio.readyState < 3) return; + + if (isPlaying) { + const video = videoRef.current; + if (video && audio.duration) { + audio.currentTime = video.currentTime % audio.duration; + } + audio.play().catch((_e) => { + /* autoplay blocked */ + }); + } else { + audio.pause(); + } + }, [isPlaying]); const clampFocusToStage = useCallback((focus: ZoomFocus, depth: ZoomDepth) => { return clampFocusToStageUtil(focus, depth, stageSizeRef.current); diff --git a/src/components/video-editor/constants.ts b/src/components/video-editor/constants.ts new file mode 100644 index 00000000..6ed0f88e --- /dev/null +++ b/src/components/video-editor/constants.ts @@ -0,0 +1,112 @@ +export interface Wallpaper { + id: string; + src: string; +} + +export interface Gradient { + id: string; + value: string; +} + +export interface AudioTrack { + id: string; + label: string; + src: string; +} + +export const WALLPAPERS: Wallpaper[] = Array.from({ length: 18 }, (_, i) => ({ + id: `wallpaper-${i + 1}`, + src: `wallpapers/wallpaper${i + 1}.jpg`, +})); + +export const GRADIENTS: Gradient[] = [ + { + id: "gradient-1", + value: + "linear-gradient( 111.6deg, rgba(114,167,232,1) 9.4%, rgba(253,129,82,1) 43.9%, rgba(253,129,82,1) 54.8%, rgba(249,202,86,1) 86.3% )", + }, + { id: "gradient-2", value: "linear-gradient(120deg, #d4fc79 0%, #96e6a1 100%)" }, + { + id: "gradient-3", + value: + "radial-gradient( circle farthest-corner at 3.2% 49.6%, rgba(80,12,139,0.87) 0%, rgba(161,10,144,0.72) 83.6% )", + }, + { + id: "gradient-4", + value: + "linear-gradient( 111.6deg, rgba(0,56,68,1) 0%, rgba(163,217,185,1) 51.5%, rgba(231, 148, 6, 1) 88.6% )", + }, + { + id: "gradient-5", + value: "linear-gradient( 107.7deg, rgba(235,230,44,0.55) 8.4%, rgba(252,152,15,1) 90.3% )", + }, + { + id: "gradient-6", + value: "linear-gradient( 91deg, rgba(72,154,78,1) 5.2%, rgba(251,206,70,1) 95.9% )", + }, + { + id: "gradient-7", + value: + "radial-gradient( circle farthest-corner at 10% 20%, rgba(2,37,78,1) 0%, rgba(4,56,126,1) 19.7%, rgba(85,245,221,1) 100.2% )", + }, + { + id: "gradient-8", + value: "linear-gradient( 109.6deg, rgba(15,2,2,1) 11.2%, rgba(36,163,190,1) 91.1% )", + }, + { id: "gradient-9", value: "linear-gradient(135deg, #FBC8B4, #2447B1)" }, + { id: "gradient-10", value: "linear-gradient(109.6deg, #F635A6, #36D860)" }, + { id: "gradient-11", value: "linear-gradient(90deg, #FF0101, #4DFF01)" }, + { id: "gradient-12", value: "linear-gradient(315deg, #EC0101, #5044A9)" }, + { id: "gradient-13", value: "linear-gradient(45deg, #ff9a9e 0%, #fad0c4 99%, #fad0c4 100%)" }, + { id: "gradient-14", value: "linear-gradient(to top, #a18cd1 0%, #fbc2eb 100%)" }, + { + id: "gradient-15", + value: + "linear-gradient(to right, #ff8177 0%, #ff867a 0%, #ff8c7f 21%, #f99185 52%, #cf556c 78%, #b12a5b 100%)", + }, + { id: "gradient-16", value: "linear-gradient(120deg, #84fab0 0%, #8fd3f4 100%)" }, + { id: "gradient-17", value: "linear-gradient(to right, #4facfe 0%, #00f2fe 100%)" }, + { + id: "gradient-18", + value: + "linear-gradient(to top, #fcc5e4 0%, #fda34b 15%, #ff7882 35%, #c8699e 52%, #7046aa 71%, #0c1db8 87%, #020f75 100%)", + }, + { id: "gradient-19", value: "linear-gradient(to right, #fa709a 0%, #fee140 100%)" }, + { id: "gradient-20", value: "linear-gradient(to top, #30cfd0 0%, #330867 100%)" }, + { id: "gradient-21", value: "linear-gradient(to top, #c471f5 0%, #fa71cd 100%)" }, + { + id: "gradient-22", + value: "linear-gradient(to right, #f78ca0 0%, #f9748f 19%, #fd868c 60%, #fe9a8b 100%)", + }, + { id: "gradient-23", value: "linear-gradient(to top, #48c6ef 0%, #6f86d6 100%)" }, + { id: "gradient-24", value: "linear-gradient(to right, #0acffe 0%, #495aff 100%)" }, +]; + +export const DEFAULT_BG_MUSIC_VOLUME = 0.3; + +// Add more presets by dropping MP3 in public/audio/ and adding an entry here +export const AUDIO_TRACKS: AudioTrack[] = [ + { + id: "upbeat-happy-corporate", + label: "Upbeat Happy Corporate", + src: "audio/upbeat-happy-corporate.mp3", + }, +]; + +export function resolveAudioTrackSrc(id: string): string { + return AUDIO_TRACKS.find((t) => t.id === id)?.src ?? `audio/${id}.mp3`; +} + +/** + * Resolve a backgroundMusic value (track ID or data URL) to a fetch-able URL. + * Used by both preview (VideoPlayback) and export (VideoEditor). + */ +export async function resolveBackgroundMusicUrl( + backgroundMusic: string, + getAssetPath: (rel: string) => Promise, +): Promise { + if (backgroundMusic.startsWith("data:")) { + return backgroundMusic; + } + return getAssetPath(resolveAudioTrackSrc(backgroundMusic)); +} diff --git a/src/components/video-editor/projectPersistence.ts b/src/components/video-editor/projectPersistence.ts index 5bb144df..aaa31dd7 100644 --- a/src/components/video-editor/projectPersistence.ts +++ b/src/components/video-editor/projectPersistence.ts @@ -2,6 +2,7 @@ import type { ExportFormat, ExportQuality, GifFrameRate, GifSizePreset } from "@ import type { ProjectMedia } from "@/lib/recordingSession"; import { normalizeProjectMedia } from "@/lib/recordingSession"; import { ASPECT_RATIOS, type AspectRatio } from "@/utils/aspectRatioUtils"; +import { DEFAULT_BG_MUSIC_VOLUME, WALLPAPERS } from "./constants"; import { type AnnotationRegion, type CropRegion, @@ -17,12 +18,7 @@ import { type ZoomRegion, } from "./types"; -const WALLPAPER_COUNT = 18; - -export const WALLPAPER_PATHS = Array.from( - { length: WALLPAPER_COUNT }, - (_, i) => `/wallpapers/wallpaper${i + 1}.jpg`, -); +export const WALLPAPER_PATHS = WALLPAPERS.map((w) => `/${w.src}`); export const PROJECT_VERSION = 2; @@ -44,6 +40,8 @@ export interface ProjectEditorState { gifFrameRate: GifFrameRate; gifLoop: boolean; gifSizePreset: GifSizePreset; + backgroundMusic: string | null; + backgroundMusicVolume: number; } export interface EditorProjectData { @@ -360,6 +358,10 @@ export function normalizeProjectEditor(editor: Partial): Pro editor.gifSizePreset === "original" ? editor.gifSizePreset : "medium", + backgroundMusic: typeof editor.backgroundMusic === "string" ? editor.backgroundMusic : null, + backgroundMusicVolume: isFiniteNumber(editor.backgroundMusicVolume) + ? clamp(editor.backgroundMusicVolume, 0, 1) + : DEFAULT_BG_MUSIC_VOLUME, }; } diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index 490eed2a..31dbe99e 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -1,4 +1,5 @@ import { WebDemuxer } from "web-demuxer"; +import { DEFAULT_BG_MUSIC_VOLUME } from "@/components/video-editor/constants"; import type { SpeedRegion, TrimRegion } from "@/components/video-editor/types"; import type { VideoMuxer } from "./muxer"; @@ -11,7 +12,7 @@ export class AudioProcessor { /** * Audio export has two modes: - * 1) no speed regions -> fast WebCodecs trim-only pipeline + * 1) no speed regions -> fast WebCodecs trim-only pipeline (with optional background music mixing) * 2) speed regions present -> pitch-preserving rendered timeline pipeline */ async process( @@ -21,6 +22,9 @@ export class AudioProcessor { trimRegions?: TrimRegion[], speedRegions?: SpeedRegion[], readEndSec?: number, + backgroundMusicUrl?: string, + effectiveDurationSeconds?: number, + backgroundMusicVolume?: number, ): Promise { const sortedTrims = trimRegions ? [...trimRegions].sort((a, b) => a.startMs - b.startMs) : []; const sortedSpeedRegions = speedRegions @@ -35,6 +39,8 @@ export class AudioProcessor { videoUrl, sortedTrims, sortedSpeedRegions, + backgroundMusicUrl, + backgroundMusicVolume, ); if (!this.cancelled) { await this.muxRenderedAudioBlob(renderedAudioBlob, muxer); @@ -43,75 +49,151 @@ export class AudioProcessor { } // No speed edits: keep the original demux/decode/encode path with trim timestamp remap. - await this.processTrimOnlyAudio(demuxer, muxer, sortedTrims, readEndSec); + await this.processTrimOnlyAudio( + demuxer, + muxer, + sortedTrims, + readEndSec, + backgroundMusicUrl, + effectiveDurationSeconds, + backgroundMusicVolume, + ); } // Legacy trim-only path. This is still used for projects without speed regions. + // Now also supports background music mixing. private async processTrimOnlyAudio( demuxer: WebDemuxer, muxer: VideoMuxer, sortedTrims: TrimRegion[], readEndSec?: number, + backgroundMusicUrl?: string, + effectiveDurationSeconds?: number, + backgroundMusicVolume?: number, ): Promise { - let audioConfig: AudioDecoderConfig; + let audioConfig: AudioDecoderConfig | null = null; try { audioConfig = (await demuxer.getDecoderConfig("audio")) as AudioDecoderConfig; } catch { - console.warn("[AudioProcessor] No audio track found, skipping"); - return; + console.warn("[AudioProcessor] No audio track found in source"); + } + + const hasSourceAudio = audioConfig !== null; + + if (hasSourceAudio) { + const codecCheck = await AudioDecoder.isConfigSupported(audioConfig!); + if (!codecCheck.supported) { + console.warn("[AudioProcessor] Audio codec not supported:", audioConfig!.codec); + audioConfig = null; + } } - const codecCheck = await AudioDecoder.isConfigSupported(audioConfig); - if (!codecCheck.supported) { - console.warn("[AudioProcessor] Audio codec not supported:", audioConfig.codec); + if (!hasSourceAudio && !backgroundMusicUrl) { + console.warn("[AudioProcessor] No audio source and no background music, skipping"); return; } + // Load background music PCM data if provided + let bgMusicBuffer: AudioBuffer | null = null; + if (backgroundMusicUrl) { + bgMusicBuffer = await this.fetchAndDecodeMusic(backgroundMusicUrl); + } + // Phase 1: Decode audio from source, skipping trimmed regions const decodedFrames: AudioData[] = []; - const decoder = new AudioDecoder({ - output: (data: AudioData) => decodedFrames.push(data), - error: (e: DOMException) => console.error("[AudioProcessor] Decode error:", e), - }); - decoder.configure(audioConfig); - - const safeReadEndSec = - typeof readEndSec === "number" && Number.isFinite(readEndSec) - ? Math.max(0, readEndSec) - : undefined; - const audioStream = ( - safeReadEndSec !== undefined - ? demuxer.read("audio", 0, safeReadEndSec) - : demuxer.read("audio") - ) as ReadableStream; - const reader = audioStream.getReader(); + if (audioConfig) { + const decoder = new AudioDecoder({ + output: (data: AudioData) => decodedFrames.push(data), + error: (e: DOMException) => console.error("[AudioProcessor] Decode error:", e), + }); + decoder.configure(audioConfig); + + const safeReadEndSec = + typeof readEndSec === "number" && Number.isFinite(readEndSec) + ? Math.max(0, readEndSec) + : undefined; + const audioStream = ( + safeReadEndSec !== undefined + ? demuxer.read("audio", 0, safeReadEndSec) + : demuxer.read("audio") + ) as ReadableStream; + const reader = audioStream.getReader(); - try { - while (!this.cancelled) { - const { done, value: chunk } = await reader.read(); - if (done || !chunk) break; + try { + while (!this.cancelled) { + const { done, value: chunk } = await reader.read(); + if (done || !chunk) break; - const timestampMs = chunk.timestamp / 1000; - if (this.isInTrimRegion(timestampMs, sortedTrims)) continue; + const timestampMs = chunk.timestamp / 1000; + if (this.isInTrimRegion(timestampMs, sortedTrims)) continue; - decoder.decode(chunk); + decoder.decode(chunk); - while (decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT && !this.cancelled) { - await new Promise((resolve) => setTimeout(resolve, 1)); + while (decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT && !this.cancelled) { + await new Promise((resolve) => setTimeout(resolve, 1)); + } + } + } finally { + try { + await reader.cancel(); + } catch { + /* reader already closed */ } } - } finally { - try { - await reader.cancel(); - } catch { - /* reader already closed */ + + if (decoder.state === "configured") { + await decoder.flush(); + decoder.close(); } } - if (decoder.state === "configured") { - await decoder.flush(); - decoder.close(); + if (this.cancelled) { + for (const f of decodedFrames) f.close(); + return; + } + + // If no source audio and no decoded frames, generate frames from background music + const sampleRate = audioConfig?.sampleRate || 48000; + const channels = audioConfig?.numberOfChannels || 2; + + if (decodedFrames.length === 0 && bgMusicBuffer) { + // Generate audio frames from background music only + const totalDurationUs = (effectiveDurationSeconds || 10) * 1_000_000; + const framesPerChunk = 960; // Standard Opus frame size at 48kHz + const chunkDurationUs = (framesPerChunk / sampleRate) * 1_000_000; + + for ( + let timestampUs = 0; + timestampUs < totalDurationUs && !this.cancelled; + timestampUs += chunkDurationUs + ) { + const bgSamples = this.getBgMusicSamples( + bgMusicBuffer, + timestampUs, + framesPerChunk, + sampleRate, + channels, + ); + + const gain = backgroundMusicVolume ?? DEFAULT_BG_MUSIC_VOLUME; + if (gain !== 1) { + for (let i = 0; i < bgSamples.length; i++) { + bgSamples[i] *= gain; + } + } + + const audioData = new AudioData({ + format: "f32-planar", + sampleRate, + numberOfFrames: framesPerChunk, + numberOfChannels: channels, + timestamp: timestampUs, + data: bgSamples.buffer as ArrayBuffer, + }); + + decodedFrames.push(audioData); + } } if (this.cancelled || decodedFrames.length === 0) { @@ -119,7 +201,7 @@ export class AudioProcessor { return; } - // Phase 2: Re-encode with timestamps adjusted for trim gaps + // Phase 2: Re-encode with timestamps adjusted for trim gaps, mixing in background music const encodedChunks: { chunk: EncodedAudioChunk; meta?: EncodedAudioChunkMetadata }[] = []; const encoder = new AudioEncoder({ @@ -129,9 +211,6 @@ export class AudioProcessor { error: (e: DOMException) => console.error("[AudioProcessor] Encode error:", e), }); - const sampleRate = audioConfig.sampleRate || 48000; - const channels = audioConfig.numberOfChannels || 2; - const encodeConfig: AudioEncoderConfig = { codec: "opus", sampleRate, @@ -155,14 +234,26 @@ export class AudioProcessor { } const timestampMs = audioData.timestamp / 1000; - const trimOffsetMs = this.computeTrimOffset(timestampMs, sortedTrims); + const trimOffsetMs = audioConfig ? this.computeTrimOffset(timestampMs, sortedTrims) : 0; const adjustedTimestampUs = audioData.timestamp - trimOffsetMs * 1000; - const adjusted = this.cloneWithTimestamp(audioData, Math.max(0, adjustedTimestampUs)); - audioData.close(); + let frameToEncode: AudioData; + + if (bgMusicBuffer && audioConfig) { + // Mix source audio with background music + frameToEncode = this.mixWithBackgroundMusic( + audioData, + bgMusicBuffer, + Math.max(0, adjustedTimestampUs), + backgroundMusicVolume ?? DEFAULT_BG_MUSIC_VOLUME, + ); + } else { + frameToEncode = this.cloneWithTimestamp(audioData, Math.max(0, adjustedTimestampUs)); + } - encoder.encode(adjusted); - adjusted.close(); + audioData.close(); + encoder.encode(frameToEncode); + frameToEncode.close(); } if (encoder.state === "configured") { @@ -187,6 +278,8 @@ export class AudioProcessor { videoUrl: string, trimRegions: TrimRegion[], speedRegions: SpeedRegion[], + backgroundMusicUrl?: string, + backgroundMusicVolume?: number, ): Promise { const media = document.createElement("audio"); media.src = videoUrl; @@ -211,6 +304,24 @@ export class AudioProcessor { const destinationNode = audioContext.createMediaStreamDestination(); sourceNode.connect(destinationNode); + // Mix background music into the same destination so MediaRecorder captures it + let bgSourceNode: AudioBufferSourceNode | null = null; + let bgGainNode: GainNode | null = null; + if (backgroundMusicUrl) { + const bgBuffer = await this.fetchAndDecodeMusic(backgroundMusicUrl); + if (bgBuffer) { + bgSourceNode = audioContext.createBufferSource(); + bgSourceNode.buffer = bgBuffer; + bgSourceNode.loop = true; + + bgGainNode = audioContext.createGain(); + bgGainNode.gain.value = backgroundMusicVolume ?? DEFAULT_BG_MUSIC_VOLUME; + + bgSourceNode.connect(bgGainNode); + bgGainNode.connect(destinationNode); + } + } + const { recorder, recordedBlobPromise } = this.startAudioRecording(destinationNode.stream); let rafId: number | null = null; @@ -220,6 +331,8 @@ export class AudioProcessor { } await this.seekTo(media, 0); + // Start background music at the same time as source playback + bgSourceNode?.start(0); await media.play(); await new Promise((resolve, reject) => { @@ -289,6 +402,18 @@ export class AudioProcessor { if (recorder.state !== "inactive") { recorder.stop(); } + // Stop and disconnect background music nodes + if (bgSourceNode) { + try { + bgSourceNode.stop(); + } catch { + /* already stopped */ + } + bgSourceNode.disconnect(); + } + if (bgGainNode) { + bgGainNode.disconnect(); + } destinationNode.stream.getTracks().forEach((track) => track.stop()); sourceNode.disconnect(); destinationNode.disconnect(); @@ -458,6 +583,129 @@ export class AudioProcessor { ); } + private async fetchAndDecodeMusic(url: string): Promise { + try { + console.log("[AudioProcessor] Fetching background music from:", url); + const response = await fetch(url); + if (!response.ok) { + console.error( + `[AudioProcessor] Background music fetch failed: ${response.status} ${response.statusText}`, + ); + return null; + } + const arrayBuffer = await response.arrayBuffer(); + console.log(`[AudioProcessor] Background music fetched: ${arrayBuffer.byteLength} bytes`); + + const audioContext = new OfflineAudioContext(2, 48000, 48000); + let audioBuffer: AudioBuffer; + try { + audioBuffer = await audioContext.decodeAudioData(arrayBuffer); + } finally { + try { + await (audioContext as unknown as AudioContext).close(); + } catch { + // OfflineAudioContext.close() is not guaranteed by spec + } + } + + console.log( + `[AudioProcessor] Background music loaded: ${audioBuffer.duration.toFixed(1)}s, ${audioBuffer.numberOfChannels}ch, ${audioBuffer.sampleRate}Hz`, + ); + return audioBuffer; + } catch (e) { + console.error("[AudioProcessor] Failed to load background music:", e); + return null; + } + } + + private getBgMusicSamples( + bgBuffer: AudioBuffer, + timestampUs: number, + numFrames: number, + targetSampleRate: number, + targetChannels: number, + ): Float32Array { + const bgSampleRate = bgBuffer.sampleRate; + const bgTotalSamples = bgBuffer.length; + const timeInSeconds = timestampUs / 1_000_000; + const bgStartSample = Math.floor((timeInSeconds * bgSampleRate) % bgTotalSamples); + + const result = new Float32Array(numFrames * targetChannels); + + for (let ch = 0; ch < targetChannels; ch++) { + const bgChannel = bgBuffer.getChannelData(Math.min(ch, bgBuffer.numberOfChannels - 1)); + for (let i = 0; i < numFrames; i++) { + const bgSampleIndex = + (bgStartSample + Math.floor((i * bgSampleRate) / targetSampleRate)) % bgTotalSamples; + // Planar layout: each channel's samples are contiguous + result[ch * numFrames + i] = bgChannel[bgSampleIndex]; + } + } + + return result; + } + + private mixWithBackgroundMusic( + src: AudioData, + bgBuffer: AudioBuffer, + newTimestamp: number, + gain: number, + ): AudioData { + const isPlanar = src.format?.includes("planar") ?? false; + const numChannels = src.numberOfChannels; + const numFrames = src.numberOfFrames; + + // Extract source samples as float32 + const srcSamples = new Float32Array(numFrames * numChannels); + if (isPlanar) { + for (let ch = 0; ch < numChannels; ch++) { + const planeSize = src.allocationSize({ planeIndex: ch }); + const plane = new ArrayBuffer(planeSize); + src.copyTo(new Uint8Array(plane), { planeIndex: ch }); + + const floatView = new Float32Array(plane); + for (let i = 0; i < numFrames && i < floatView.length; i++) { + srcSamples[ch * numFrames + i] = floatView[i]; + } + } + } else { + const totalSize = src.allocationSize({ planeIndex: 0 }); + const buf = new ArrayBuffer(totalSize); + src.copyTo(new Uint8Array(buf), { planeIndex: 0 }); + const floatView = new Float32Array(buf); + // Convert interleaved to planar + for (let i = 0; i < numFrames; i++) { + for (let ch = 0; ch < numChannels; ch++) { + srcSamples[ch * numFrames + i] = floatView[i * numChannels + ch] || 0; + } + } + } + + // Get background music samples at the correct position + const bgSamples = this.getBgMusicSamples( + bgBuffer, + newTimestamp, + numFrames, + src.sampleRate, + numChannels, + ); + + // Mix: source + background at reduced gain + const mixed = new Float32Array(numFrames * numChannels); + for (let i = 0; i < mixed.length; i++) { + mixed[i] = Math.max(-1, Math.min(1, srcSamples[i] + bgSamples[i] * gain)); + } + + return new AudioData({ + format: "f32-planar", + sampleRate: src.sampleRate, + numberOfFrames: numFrames, + numberOfChannels: numChannels, + timestamp: newTimestamp, + data: mixed.buffer as ArrayBuffer, + }); + } + private cloneWithTimestamp(src: AudioData, newTimestamp: number): AudioData { const isPlanar = src.format?.includes("planar") ?? false; const numPlanes = isPlanar ? src.numberOfChannels : 1; diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index c80d4700..23a78496 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -30,6 +30,8 @@ interface VideoExporterConfig extends ExportConfig { annotationRegions?: AnnotationRegion[]; previewWidth?: number; previewHeight?: number; + backgroundMusicUrl?: string; + backgroundMusicVolume?: number; onProgress?: (progress: ExportProgress) => void; } @@ -97,8 +99,8 @@ export class VideoExporter { // Initialize video encoder await this.initializeEncoder(); - // Initialize muxer (with audio if source has an audio track) - const hasAudio = videoInfo.hasAudio; + // Initialize muxer (with audio if source has an audio track or background music) + const hasAudio = videoInfo.hasAudio || !!this.config.backgroundMusicUrl; this.muxer = new VideoMuxer(this.config, hasAudio); await this.muxer.initialize(); @@ -257,7 +259,7 @@ export class VideoExporter { }); } - // Process audio track if present + // Process audio track if present (or if background music is set) if (hasAudio && !this.cancelled) { const demuxer = this.streamingDecoder!.getDemuxer(); if (demuxer) { @@ -270,6 +272,9 @@ export class VideoExporter { this.config.trimRegions, this.config.speedRegions, readEndSec, + this.config.backgroundMusicUrl, + effectiveDuration, + this.config.backgroundMusicVolume, ); } }