From d8cebfce59be3cfc31418841c1594c107dacdd27 Mon Sep 17 00:00:00 2001
From: edisonmoy <moyedison@gmail.com>
Date: Tue, 17 Dec 2024 10:19:12 -0500
Subject: [PATCH] Convert mp4 to webm for streaming audio

---
 .../lib/media/audioRecorder.ts                | 56 ++++++++++++++++++-
 .../package-lock.json                         | 32 +++++++++++
 typescript-streaming-sandbox/package.json     |  2 +
 3 files changed, 87 insertions(+), 3 deletions(-)

diff --git a/typescript-streaming-sandbox/lib/media/audioRecorder.ts b/typescript-streaming-sandbox/lib/media/audioRecorder.ts
index d06687cc..c651ae5b 100644
--- a/typescript-streaming-sandbox/lib/media/audioRecorder.ts
+++ b/typescript-streaming-sandbox/lib/media/audioRecorder.ts
@@ -1,4 +1,7 @@
 import { sleep } from "../utilities/asyncUtilities";
+import { fetchFile } from "@ffmpeg/util";
+
+let ffmpeg: any = null;
 
 export class AudioRecorder {
   private recorder;
@@ -9,13 +12,54 @@ export class AudioRecorder {
     this.mediaStream = mediaStream;
   }
 
+  private static async convertAudio(blob: Blob): Promise<Blob> {
+    await ffmpeg.load();
+
+    const inputName = "input.mp4";
+    const outputName = "output.webm";
+
+    ffmpeg.writeFile(inputName, await fetchFile(blob));
+
+    await ffmpeg.exec(["-i", inputName, "-c:a", "libopus", outputName]);
+
+    const data = await ffmpeg.readFile(outputName);
+    return new Blob([data], { type: "audio/webm" });
+  }
+
   static async create(): Promise<AudioRecorder> {
+    await AudioRecorder.loadFFmpeg();
     const mediaOptions = { video: false, audio: true };
     const mediaStream = await navigator.mediaDevices.getUserMedia(mediaOptions);
-    const recorder = new MediaRecorder(mediaStream);
+    const mimeType = AudioRecorder.getSupportedMimeType();
+    const recorder = new MediaRecorder(mediaStream, { mimeType });
     return new AudioRecorder(recorder, mediaStream);
   }
 
+  private static getSupportedMimeType(): string {
+    const mimeTypes = ["audio/webm", "audio/webm;codecs=opus", "audio/mp4", "audio/mp4;codecs=aac"];
+
+    for (const mimeType of mimeTypes) {
+      if (MediaRecorder.isTypeSupported(mimeType)) {
+        return mimeType;
+      }
+    }
+    console.warn("No supported MIME type found. Defaulting to audio/webm.");
+    return "audio/webm";
+  }
+
+  private static async loadFFmpeg() {
+    if (typeof window === "undefined") return;
+    if (!ffmpeg) {
+      const { FFmpeg } = await import("@ffmpeg/ffmpeg");
+      const { fetchFile } = await import("@ffmpeg/util");
+
+      ffmpeg = new FFmpeg();
+      ffmpeg.fetchFile = fetchFile;
+
+      await ffmpeg.load();
+    }
+  }
+
   async stopRecording() {
     this.mediaStream.getTracks().forEach((track) => {
       track.stop();
@@ -24,8 +68,14 @@ export class AudioRecorder {
 
   record(length: number): Promise<Blob> {
     return new Promise(async (resolve: (blob: Blob) => void, _) => {
-      this.recorder.ondataavailable = (blobEvent) => {
-        resolve(blobEvent.data);
+      this.recorder.ondataavailable = async (blobEvent) => {
+        let recordedBlob = blobEvent.data;
+
+        if (recordedBlob.type === "audio/mp4") {
+          recordedBlob = await AudioRecorder.convertAudio(recordedBlob);
+        }
+
+        resolve(recordedBlob);
       };
 
       if (this.recorder.state !== "recording") this.recorder.start();
diff --git a/typescript-streaming-sandbox/package-lock.json b/typescript-streaming-sandbox/package-lock.json
index 61ded8f8..43e26afe 100644
--- a/typescript-streaming-sandbox/package-lock.json
+++ b/typescript-streaming-sandbox/package-lock.json
@@ -8,6 +8,8 @@
       "name": "sandbox",
       "version": "0.2.0",
       "dependencies": {
+        "@ffmpeg/ffmpeg": "^0.12.10",
+        "@ffmpeg/util": "^0.12.1",
         "@fontsource/poppins": "^4.5.10",
         "@phosphor-icons/react": "2.0.5",
         "@types/node": "18.11.9",
@@ -39,6 +41,36 @@
         "node": ">=6.9.0"
       }
     },
+    "node_modules/@ffmpeg/ffmpeg": {
+      "version": "0.12.10",
+      "resolved": "https://registry.npmjs.org/@ffmpeg/ffmpeg/-/ffmpeg-0.12.10.tgz",
+      "integrity": "sha512-lVtk8PW8e+NUzGZhPTWj2P1J4/NyuCrbDD3O9IGpSeLYtUZKBqZO8CNj1WYGghep/MXoM8e1qVY1GztTkf8YYQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@ffmpeg/types": "^0.12.2"
+      },
+      "engines": {
+        "node": ">=18.x"
+      }
+    },
+    "node_modules/@ffmpeg/types": {
+      "version": "0.12.2",
+      "resolved": "https://registry.npmjs.org/@ffmpeg/types/-/types-0.12.2.tgz",
+      "integrity": "sha512-NJtxwPoLb60/z1Klv0ueshguWQ/7mNm106qdHkB4HL49LXszjhjCCiL+ldHJGQ9ai2Igx0s4F24ghigy//ERdA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=16.x"
+      }
+    },
+    "node_modules/@ffmpeg/util": {
+      "version": "0.12.1",
+      "resolved": "https://registry.npmjs.org/@ffmpeg/util/-/util-0.12.1.tgz",
+      "integrity": "sha512-10jjfAKWaDyb8+nAkijcsi9wgz/y26LOc1NKJradNMyCIl6usQcBbhkjX5qhALrSBcOy6TOeksunTYa+a03qNQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=18.x"
+      }
+    },
     "node_modules/@fontsource/poppins": {
       "version": "4.5.10",
       "license": "MIT"
diff --git a/typescript-streaming-sandbox/package.json b/typescript-streaming-sandbox/package.json
index 8daa91c4..eb68430f 100644
--- a/typescript-streaming-sandbox/package.json
+++ b/typescript-streaming-sandbox/package.json
@@ -9,6 +9,8 @@
     "lint": "next lint"
   },
   "dependencies": {
+    "@ffmpeg/ffmpeg": "^0.12.10",
+    "@ffmpeg/util": "^0.12.1",
     "@fontsource/poppins": "^4.5.10",
     "@phosphor-icons/react": "2.0.5",
     "@types/node": "18.11.9",