diff --git a/package.json b/package.json index 64910dda..8dab1b4b 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,8 @@ "esbuild": "^0.25.0", "esbuild-plugin-tailwindcss": "^2.0.1", "framer-motion": "^12.4.9", + "i18next": "^23.16.8", + "i18next-browser-languagedetector": "^8.2.0", "katex": "^0.16.21", "langgraph-nextjs-api-passthrough": "^0.0.4", "lodash": "^4.17.21", @@ -43,8 +45,10 @@ "nuqs": "^2.4.1", "react": "^19.0.0", "react-dom": "^19.0.0", + "react-i18next": "^15.6.0", "react-markdown": "^10.0.1", "react-syntax-highlighter": "^15.5.0", + "react-use-websocket": "^4.13.0", "recharts": "^2.15.1", "rehype-katex": "^7.0.1", "remark-gfm": "^4.0.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 030fd063..5925b88d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -56,6 +56,12 @@ importers: framer-motion: specifier: ^12.4.9 version: 12.12.1(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + i18next: + specifier: ^23.16.8 + version: 23.16.8 + i18next-browser-languagedetector: + specifier: ^8.2.0 + version: 8.2.0 katex: specifier: ^0.16.21 version: 0.16.22 @@ -80,12 +86,18 @@ importers: react-dom: specifier: ^19.0.0 version: 19.1.0(react@19.1.0) + react-i18next: + specifier: ^15.6.0 + version: 15.6.0(i18next@23.16.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(typescript@5.7.3) react-markdown: specifier: ^10.0.1 version: 10.1.0(@types/react@19.1.4)(react@19.1.0) react-syntax-highlighter: specifier: ^15.5.0 version: 15.6.1(react@19.1.0) + react-use-websocket: + specifier: ^4.13.0 + version: 4.13.0 recharts: specifier: ^2.15.1 version: 2.15.3(react-dom@19.1.0(react@19.1.0))(react@19.1.0) @@ -201,6 +213,10 @@ packages: resolution: {integrity: sha512-1x3D2xEk2fRo3PAhwQwu5UubzgiVWSXTBfWpVd2Mx2AzRqJuDJCsgaDVZ7HB5iGzDW1Hl1sWN2mFyKjmR9uAog==} engines: {node: '>=6.9.0'} + '@babel/runtime@7.27.6': + resolution: {integrity: sha512-vbavdySgbTTrmFE+EsiqUTzlOr5bzlnJtUv9PynGCAKvfQqjIXbvFdumPM/GxMDfyuGMJaJAU6TO4zc1Jf1i8Q==} + engines: {node: '>=6.9.0'} + '@cfworker/json-schema@4.1.1': resolution: {integrity: sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==} @@ -2123,12 +2139,21 @@ packages: highlightjs-vue@1.0.0: resolution: {integrity: sha512-PDEfEF102G23vHmPhLyPboFCD+BkMGu+GuJe2d9/eH4FsCwvgBpnc9n0pGE+ffKdph38s6foEZiEjdgHdzp+IA==} + html-parse-stringify@3.0.1: + resolution: {integrity: sha512-KknJ50kTInJ7qIScF3jeaFRpMpE8/lfiTdzf/twXyPBLAGrLRTmkz3AdTnKeh40X8k9L2fdYwEp/42WGXIRGcg==} + html-url-attributes@3.0.1: resolution: {integrity: sha512-ol6UPyBWqsrO6EJySPz2O7ZSr856WDrEzM5zMqp+FJJLGMW35cLYmmZnl0vztAZxRUoNZJFTCohfjuIJ8I4QBQ==} humanize-ms@1.2.1: resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==} + i18next-browser-languagedetector@8.2.0: + resolution: {integrity: sha512-P+3zEKLnOF0qmiesW383vsLdtQVyKtCNA9cjSoKCppTKPQVfKd2W8hbVo5ZhNJKDqeM7BOcvNoKJOjpHh4Js9g==} + + i18next@23.16.8: + resolution: {integrity: sha512-06r/TitrM88Mg5FdUXAKL96dJMzgqLE5dv3ryBAra4KCwD9mJ4ndOTS95ZuymIGoE+2hzfdaMak2X11/es7ZWg==} + icss-utils@5.1.0: resolution: {integrity: sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA==} engines: {node: ^10 || ^12 || >= 14} @@ -3006,6 +3031,22 @@ packages: peerDependencies: react: ^19.1.0 + react-i18next@15.6.0: + resolution: {integrity: sha512-W135dB0rDfiFmbMipC17nOhGdttO5mzH8BivY+2ybsQBbXvxWIwl3cmeH3T9d+YPBSJu/ouyJKFJTtkK7rJofw==} + peerDependencies: + i18next: '>= 23.2.3' + react: '>= 16.8.0' + react-dom: '*' + react-native: '*' + typescript: ^5 + peerDependenciesMeta: + react-dom: + optional: true + react-native: + optional: true + typescript: + optional: true + react-is@16.13.1: resolution: {integrity: sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==} @@ -3065,6 +3106,9 @@ packages: react: '>=16.6.0' react-dom: '>=16.6.0' + react-use-websocket@4.13.0: + resolution: {integrity: sha512-anMuVoV//g2N76Wxqvqjjo1X48r9Np3y1/gMl7arX84tAPXdy5R7sB5lO5hvCzQRYjqXwV8XMAiEBOUbyrZFrw==} + react@19.1.0: resolution: {integrity: sha512-FS+XFBNvn3GTAWq26joslQgWNoFu08F4kl0J4CgdNKADkdSGXQyTCnKteIAJy96Br6YbpEU1LSzV5dYtjMkMDg==} engines: {node: '>=0.10.0'} @@ -3484,6 +3528,10 @@ packages: victory-vendor@36.9.2: resolution: {integrity: sha512-PnpQQMuxlwYdocC8fIJqVXvkeViHYzotI+NJrCuav0ZYFoq912ZHBk3mCeuj+5/VpodOjPe1z0Fk2ihgzlXqjQ==} + void-elements@3.1.0: + resolution: {integrity: sha512-Dhxzh5HZuiHQhbvTW9AMetFfBHDMYpo23Uo9btPXgdYP+3T5S+p+jgNy7spra+veYhBP2dCSgxR/i2Y02h5/6w==} + engines: {node: '>=0.10.0'} + web-namespaces@2.0.1: resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==} @@ -3568,6 +3616,8 @@ snapshots: '@babel/runtime@7.27.1': {} + '@babel/runtime@7.27.6': {} + '@cfworker/json-schema@4.1.1': {} '@emnapi/core@1.4.3': @@ -5535,6 +5585,10 @@ snapshots: highlightjs-vue@1.0.0: {} + html-parse-stringify@3.0.1: + dependencies: + void-elements: 3.1.0 + html-url-attributes@3.0.1: {} humanize-ms@1.2.1: @@ -5542,6 +5596,14 @@ snapshots: ms: 2.1.3 optional: true + i18next-browser-languagedetector@8.2.0: + dependencies: + '@babel/runtime': 7.27.1 + + i18next@23.16.8: + dependencies: + '@babel/runtime': 7.27.1 + icss-utils@5.1.0(postcss@8.5.3): dependencies: postcss: 8.5.3 @@ -6566,6 +6628,16 @@ snapshots: react: 19.1.0 scheduler: 0.26.0 + react-i18next@15.6.0(i18next@23.16.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(typescript@5.7.3): + dependencies: + '@babel/runtime': 7.27.6 + html-parse-stringify: 3.0.1 + i18next: 23.16.8 + react: 19.1.0 + optionalDependencies: + react-dom: 19.1.0(react@19.1.0) + typescript: 5.7.3 + react-is@16.13.1: {} react-is@18.3.1: {} @@ -6642,6 +6714,8 @@ snapshots: react: 19.1.0 react-dom: 19.1.0(react@19.1.0) + react-use-websocket@4.13.0: {} + react@19.1.0: {} recharts-scale@0.4.5: @@ -7231,6 +7305,8 @@ snapshots: d3-time: 3.1.0 d3-timer: 3.0.1 + void-elements@3.1.0: {} + web-namespaces@2.0.1: {} web-streams-polyfill@4.0.0-beta.3: diff --git a/public/audio-playback-worklet.js b/public/audio-playback-worklet.js new file mode 100644 index 00000000..bf041088 --- /dev/null +++ b/public/audio-playback-worklet.js @@ -0,0 +1,33 @@ +class AudioPlaybackWorklet extends AudioWorkletProcessor { + constructor() { + super(); + this.port.onmessage = this.handleMessage.bind(this); + this.buffer = []; + } + + handleMessage(event) { + if (event.data === null) { + this.buffer = []; + return; + } + this.buffer.push(...event.data); + } + + process(inputs, outputs, parameters) { + const output = outputs[0]; + const channel = output[0]; + + if (this.buffer.length > channel.length) { + const toProcess = this.buffer.slice(0, channel.length); + this.buffer = this.buffer.slice(channel.length); + channel.set(toProcess.map(v => v / 32768)); + } else { + channel.set(this.buffer.map(v => v / 32768)); + this.buffer = []; + } + + return true; + } +} + +registerProcessor("audio-playback-worklet", AudioPlaybackWorklet); diff --git a/public/audio-processor-worklet.js b/public/audio-processor-worklet.js new file mode 100644 index 00000000..5919d955 --- /dev/null +++ b/public/audio-processor-worklet.js @@ -0,0 +1,30 @@ +const MIN_INT16 = -0x8000; +const MAX_INT16 = 0x7fff; + +class PCMAudioProcessor extends AudioWorkletProcessor { + constructor() { + super(); + } + + process(inputs, outputs, parameters) { + const input = inputs[0]; + if (input.length > 0) { + const float32Buffer = input[0]; + const int16Buffer = this.float32ToInt16(float32Buffer); + this.port.postMessage(int16Buffer); + } + return true; + } + + float32ToInt16(float32Array) { + const int16Array = new Int16Array(float32Array.length); + for (let i = 0; i < float32Array.length; i++) { + let val = Math.floor(float32Array[i] * MAX_INT16); + val = Math.max(MIN_INT16, Math.min(MAX_INT16, val)); + int16Array[i] = val; + } + return int16Array; + } +} + +registerProcessor("audio-processor-worklet", PCMAudioProcessor); diff --git a/src/app/favicon.ico b/src/app/favicon.ico index 2a7a6db0..f9e1bdd5 100644 Binary files a/src/app/favicon.ico and b/src/app/favicon.ico differ diff --git a/src/app/layout.tsx b/src/app/layout.tsx index af31f8e5..716a63a6 100644 --- a/src/app/layout.tsx +++ b/src/app/layout.tsx @@ -11,7 +11,7 @@ const inter = Inter({ }); export const metadata: Metadata = { - title: "Agent Chat", + title: "EchOS Chat", description: "Agent Chat UX by LangChain", }; diff --git a/src/components/audio/player.ts b/src/components/audio/player.ts new file mode 100644 index 00000000..ff2ab1c3 --- /dev/null +++ b/src/components/audio/player.ts @@ -0,0 +1,32 @@ +export class Player { + private playbackNode: AudioWorkletNode | null = null; + private audioContext: AudioContext | null = null; + + async init(sampleRate: number) { + this.audioContext = new AudioContext({ sampleRate }); + await this.audioContext.audioWorklet.addModule("/audio-playback-worklet.js"); + + this.playbackNode = new AudioWorkletNode(this.audioContext, "audio-playback-worklet"); + this.playbackNode.connect(this.audioContext.destination); + } + + play(buffer: Int16Array) { + if (this.playbackNode) { + this.playbackNode.port.postMessage(buffer); + } + } + + stop() { + if (this.playbackNode) { + this.playbackNode.port.postMessage(null); + } + } + + async close() { + if (this.audioContext) { + await this.audioContext.close(); + this.audioContext = null; + } + this.playbackNode = null; + } +} diff --git a/src/components/audio/recorder.ts b/src/components/audio/recorder.ts new file mode 100644 index 00000000..0e06c7c6 --- /dev/null +++ b/src/components/audio/recorder.ts @@ -0,0 +1,52 @@ +export class Recorder { + onDataAvailable: (buffer: Iterable) => void; + private audioContext: AudioContext | null = null; + private mediaStream: MediaStream | null = null; + private mediaStreamSource: MediaStreamAudioSourceNode | null = null; + private workletNode: AudioWorkletNode | null = null; + + public constructor(onDataAvailable: (buffer: Iterable) => void) { + this.onDataAvailable = onDataAvailable; + } + + async start(stream: MediaStream) { + try { + if (this.audioContext) { + await this.audioContext.close(); + } + + this.audioContext = new AudioContext({ sampleRate: 24000 }); + + await this.audioContext.audioWorklet.addModule("/audio-processor-worklet.js"); + + this.mediaStream = stream; + this.mediaStreamSource = this.audioContext.createMediaStreamSource(this.mediaStream); + + this.workletNode = new AudioWorkletNode(this.audioContext, "audio-processor-worklet"); + this.workletNode.port.onmessage = event => { + this.onDataAvailable(event.data.buffer); + }; + + this.mediaStreamSource.connect(this.workletNode); + this.workletNode.connect(this.audioContext.destination); + } catch (error) { + console.error("Error starting audio recorder:", error); + this.stop(); + } + } + + async stop() { + if (this.mediaStream) { + this.mediaStream.getTracks().forEach(track => track.stop()); + this.mediaStream = null; + } + + if (this.audioContext) { + await this.audioContext.close(); + this.audioContext = null; + } + + this.mediaStreamSource = null; + this.workletNode = null; + } +} diff --git a/src/components/thread/index.tsx b/src/components/thread/index.tsx index d52a1594..2aefabec 100644 --- a/src/components/thread/index.tsx +++ b/src/components/thread/index.tsx @@ -22,7 +22,9 @@ import { SquarePen, XIcon, Plus, - CircleX, + Mic, + Check, + X, } from "lucide-react"; import { useQueryState, parseAsBoolean } from "nuqs"; import { StickToBottom, useStickToBottomContext } from "use-stick-to-bottom"; @@ -46,6 +48,9 @@ import { ArtifactTitle, useArtifactContext, } from "./artifact"; +import VoiceChat from "../voice-chat"; +import useAudioRecorder from "@/hooks/useAudioRecorder"; +import { useCallback } from "react"; function StickyToBottomContent(props: { content: ReactNode; @@ -126,6 +131,8 @@ export function Thread() { parseAsBoolean.withDefault(false), ); const [input, setInput] = useState(""); + const [voiceChatOpen, setVoiceChatOpen] = useState(false); + const [voiceModeActive, setVoiceModeActive] = useState(false); const { contentBlocks, setContentBlocks, @@ -139,6 +146,214 @@ export function Thread() { const [firstTokenReceived, setFirstTokenReceived] = useState(false); const isLargeScreen = useMediaQuery("(min-width: 1024px)"); + // Dictation state + const [isDictating, setIsDictating] = useState(false); + const [dictationTranscript, setDictationTranscript] = useState(""); + const recognitionRef = useRef(null); + const [showWaveform, setShowWaveform] = useState(false); + const canvasRef = useRef(null); + const audioContextRef = useRef(null); + const analyserRef = useRef(null); + const animationFrameRef = useRef(null); + const sourceRef = useRef(null); + const streamRef = useRef(null); + + // --- Scrolling waveform buffer --- + const barWidth = 2; + const barSpacing = 2; + const barColor = 'rgb(77, 74, 74)'; + const idleDotHeight = 4; + const idleDotRadius = 1.2; + const waveformLength = 100; + + const waveformBufferRef = useRef([]); // stores bar heights + + // Helper: get amplitude from dataArray + function getAmplitude(dataArray: Uint8Array) { + // Use peak (max absolute deviation from 128) for a more responsive bar + let peak = 0; + for (let i = 0; i < dataArray.length; i++) { + const deviation = Math.abs(dataArray[i] - 128); + if (deviation > peak) peak = deviation; + } + return peak / 128; // 0..1 + } + + // Overwrite drawWaveform for scrolling effect + const drawWaveform = useCallback(() => { + if (!canvasRef.current || !analyserRef.current) { + return; + } + const canvas = canvasRef.current; + const ctx = canvas.getContext('2d'); + if (!ctx) return; + const analyser = analyserRef.current; + const bufferLength = analyser.fftSize; + const dataArray = new Uint8Array(bufferLength); + analyser.getByteTimeDomainData(dataArray); + + // Get amplitude (0..1) + const amplitude = getAmplitude(dataArray); + // Decide if we are idle (no voice) or active (voice) + const isActive = amplitude > 0.05; // threshold for voice + // Compute new bar height + let newBarHeight; + if (isActive) { + newBarHeight = Math.max(8, amplitude * canvas.height * 0.9); + } else { + newBarHeight = idleDotHeight; + } + // Update buffer: shift left, push new value + let buffer = waveformBufferRef.current; + if (buffer.length < waveformLength) { + // Fill with idle dots initially + buffer = Array(waveformLength - buffer.length).fill(idleDotHeight).concat(buffer); + } + buffer.push(newBarHeight); + if (buffer.length > waveformLength) buffer.shift(); + waveformBufferRef.current = buffer; + + // Draw + ctx.clearRect(0, 0, canvas.width, canvas.height); + for (let i = 0; i < buffer.length; i++) { + const x = i * (barWidth + barSpacing); + const h = buffer[i]; + const y = (canvas.height - h) / 2; + ctx.fillStyle = barColor; + if (h === idleDotHeight) { + // Draw dot + ctx.beginPath(); + ctx.arc(x + barWidth / 2, canvas.height / 2, idleDotRadius, 0, 2 * Math.PI); + ctx.fill(); + } else { + // Draw bar + ctx.fillRect(x, y, barWidth, h); + } + } + animationFrameRef.current = requestAnimationFrame(drawWaveform); + }, []); + + // Start waveform + const startWaveform = useCallback(async () => { + try { + console.log('Requesting microphone for waveform...'); + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + streamRef.current = stream; + const audioContext = new window.AudioContext(); + audioContextRef.current = audioContext; + const analyser = audioContext.createAnalyser(); + analyser.fftSize = 2048; + analyserRef.current = analyser; + const source = audioContext.createMediaStreamSource(stream); + sourceRef.current = source; + source.connect(analyser); + setShowWaveform(true); + console.log('Microphone stream and analyser set up, ready for waveform'); + // drawWaveform(); // REMOVE this direct call + } catch (err) { + console.error('Could not access microphone for waveform.', err); + toast.error('Could not access microphone for waveform.'); + } + }, [drawWaveform]); + + // Start drawing waveform only when both canvas and analyser are available + useEffect(() => { + if (showWaveform && canvasRef.current && analyserRef.current) { + console.log('Starting waveform animation'); + drawWaveform(); + } + }, [showWaveform, drawWaveform]); + + // On stopWaveform, clear buffer + const stopWaveform = useCallback(() => { + setShowWaveform(false); + if (animationFrameRef.current) { + cancelAnimationFrame(animationFrameRef.current); + animationFrameRef.current = null; + } + if (audioContextRef.current) { + audioContextRef.current.close(); + audioContextRef.current = null; + } + if (streamRef.current) { + streamRef.current.getTracks().forEach(track => track.stop()); + streamRef.current = null; + } + analyserRef.current = null; + sourceRef.current = null; + if (canvasRef.current) { + const ctx = canvasRef.current.getContext('2d'); + if (ctx) ctx.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height); + } + waveformBufferRef.current = []; + }, []); + + // Dictation handler using Web Speech API (manual stop/cancel) + const handleDictateClick = async () => { + if (!isDictating) { + setIsDictating(true); + setDictationTranscript(""); + if (!('webkitSpeechRecognition' in window || 'SpeechRecognition' in window)) { + toast.error('Speech recognition is not supported in this browser.'); + setIsDictating(false); + return; + } + await startWaveform(); + const SpeechRecognition = (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition; + const recognition = new SpeechRecognition(); + recognitionRef.current = recognition; + recognition.continuous = true; // keep listening + recognition.interimResults = true; // accumulate transcript + recognition.lang = 'en-US'; + recognition.onresult = (event: any) => { + let transcript = ""; + for (let i = 0; i < event.results.length; i++) { + transcript += event.results[i][0].transcript; + } + setDictationTranscript(transcript); + }; + recognition.onerror = (event: any) => { + toast.error('Dictation error: ' + event.error); + }; + recognition.onend = () => { + // Do not auto-stop; restart if still dictating + if (isDictating) { + recognition.start(); + } + }; + recognition.start(); + } else { + // Should not happen, but stop if already dictating + setIsDictating(false); + if (recognitionRef.current) { + recognitionRef.current.stop(); + } + stopWaveform(); + setDictationTranscript(""); + } + }; + + // Stop dictation and insert transcript + const handleDictationStop = () => { + setIsDictating(false); + if (recognitionRef.current) { + recognitionRef.current.stop(); + } + stopWaveform(); + setInput((prev) => prev + (prev ? ' ' : '') + dictationTranscript.trim()); + setDictationTranscript(""); + }; + + // Cancel dictation + const handleDictationCancel = () => { + setIsDictating(false); + if (recognitionRef.current) { + recognitionRef.current.stop(); + } + stopWaveform(); + setDictationTranscript(""); + }; + const stream = useStreamContext(); const messages = stream.messages; const isLoading = stream.isLoading; @@ -252,6 +467,31 @@ export function Thread() { (m) => m.type === "ai" || m.type === "tool", ); + // Add a callback to post a message from VoiceChat to the main chat UI, with type + const postVoiceChatMessage = (text: string, type: 'human' | 'ai') => { + if (!text.trim() || isLoading) return; + const newMessage: Message = { + id: uuidv4(), + type, + content: [ + { type: "text", text }, + ], + }; + // Prevent duplicate keys: only add if id is not already present + const existingIds = new Set((stream.messages ?? []).map(m => m.id)); + if (existingIds.has(newMessage.id)) return; + stream.submit( + { messages: [...stream.messages, newMessage] }, + { + streamMode: ["values"], + optimisticValues: (prev) => ({ + ...prev, + messages: [...(prev.messages ?? []), newMessage], + }), + }, + ); + }; + return (
@@ -260,14 +500,22 @@ export function Thread() { style={{ width: 300 }} animate={ isLargeScreen - ? { x: chatHistoryOpen ? 0 : -300 } - : { x: chatHistoryOpen ? 0 : -300 } + ? { + x: chatHistoryOpen ? 0 : -300, + opacity: 1, + filter: "blur(0px)" + } + : { + x: chatHistoryOpen ? 0 : -300, + opacity: 1, + filter: "blur(0px)" + } } initial={{ x: -300 }} transition={ isLargeScreen ? { type: "spring", stiffness: 300, damping: 30 } - : { duration: 0 } + : { duration: 0.5, ease: "easeInOut" } } >
{!chatStarted && ( @@ -357,12 +609,12 @@ export function Thread() { damping: 30, }} > - + /> */} - Agent Chat + EchOS Chat
@@ -433,9 +685,8 @@ export function Thread() {
{!chatStarted && (
-

- Agent Chat + EchOS Chat

)} @@ -451,91 +702,147 @@ export function Thread() { : "border border-solid", )} > -
- -