diff --git a/deploy/parakeet-stt/README.md b/deploy/parakeet-stt/README.md new file mode 100644 index 0000000..c0ead19 --- /dev/null +++ b/deploy/parakeet-stt/README.md @@ -0,0 +1,57 @@ +# parakeet.cpp STT — deploy artifacts + +Resident parakeet.cpp STT that drop-in replaces whisper-server on +`127.0.0.1:8178` (same `POST /inference` → `{"text": ...}` contract, so +genie-core needs no change). See [`../../docs/stt-parakeet-evaluation.md`](../../docs/stt-parakeet-evaluation.md) +for the whisper-vs-parakeet comparison and rationale. + +Contents: +- `serve-mode.patch` — adds a resident `serve` subcommand to parakeet.cpp's CLI + (load model once, transcribe wav paths read from stdin). Against `v0.1.1`. +- `parakeet-stt-server.py` — HTTP shim: spawns `parakeet-cli serve` once and + serves `POST /inference` on `:8178`; resamples input to 16 kHz mono via `sox`. +- `genie-parakeet.service` — systemd unit (resident, `Restart=always`, on boot). + +## Build (on the Jetson) + +```sh +# 1. parakeet.cpp v0.1.1, native CUDA build (sm_87) +git clone --recursive --branch v0.1.1 https://github.com/mudler/parakeet.cpp ~/parakeet.cpp +cd ~/parakeet.cpp +git apply /path/to/serve-mode.patch # adds the `serve` subcommand +export PATH=/usr/local/cuda/bin:$PATH +cmake -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON \ + -DPARAKEET_GGML_CUDA=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc \ + -DCMAKE_CUDA_ARCHITECTURES=87 -DPARAKEET_BUILD_CLI=ON -DPARAKEET_BUILD_TESTS=OFF +cmake --build build -j4 --target parakeet-cli + +# 2. model (lean, near-lossless q8_0) +mkdir -p ~/parakeet-models +curl -fL -o ~/parakeet-models/tdt_ctc-110m-q8_0.gguf \ + https://huggingface.co/mudler/parakeet-cpp-gguf/resolve/main/tdt_ctc-110m-q8_0.gguf +``` + +## Install (swap whisper → parakeet) + +```sh +sudo install -m644 genie-parakeet.service /etc/systemd/system/ +install -m755 parakeet-stt-server.py ~/parakeet-stt-server.py +sudo systemctl daemon-reload +sudo systemctl disable --now genie-whisper # free GPU + :8178 +sudo systemctl enable --now genie-parakeet # parakeet now answers :8178 +curl -s http://127.0.0.1:8178/ # {"engine":"parakeet-resident", ...} +``` + +## Revert (back to whisper) + +```sh +sudo systemctl disable --now genie-parakeet +sudo systemctl enable --now genie-whisper +``` + +## Tunables + +- `PK_MODEL` (service env) — swap the model, e.g. `tdt-0.6b-v2-q8_0.gguf` for more + accuracy at higher footprint. +- `PK_DECODER` — `tdt` (default) or `ctc`. +- Runtime needs `LD_LIBRARY_PATH=/usr/local/cuda/lib64` (set in the unit). diff --git a/deploy/parakeet-stt/genie-parakeet.service b/deploy/parakeet-stt/genie-parakeet.service new file mode 100644 index 0000000..45ab5f7 --- /dev/null +++ b/deploy/parakeet-stt/genie-parakeet.service @@ -0,0 +1,19 @@ +[Unit] +Description=GeniePod Parakeet STT Server (parakeet.cpp resident; replaces whisper on :8178) +After=genie-ai-runtime.service network.target +Wants=genie-ai-runtime.service + +[Service] +Type=simple +User=aihpc +Environment=PK_PORT=8178 +Environment=PK_DECODER=tdt +Environment=PK_MODEL=/home/aihpc/parakeet-models/tdt_ctc-110m-q8_0.gguf +Environment=PATH=/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +Environment=LD_LIBRARY_PATH=/usr/local/cuda/lib64 +ExecStart=/usr/bin/python3 /home/aihpc/parakeet-stt-server.py +Restart=always +RestartSec=3 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/parakeet-stt/parakeet-stt-server.py b/deploy/parakeet-stt/parakeet-stt-server.py new file mode 100644 index 0000000..827aacf --- /dev/null +++ b/deploy/parakeet-stt/parakeet-stt-server.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Resident parakeet STT server: drop-in for whisper.cpp's POST /inference on :8178. +Spawns `parakeet-cli serve` once (model stays loaded) and pipes each request to it.""" +import http.server, socketserver, subprocess, tempfile, os, json, cgi, threading, sys, time, glob +BIN=os.path.expanduser("~/parakeet.cpp/build/examples/cli/parakeet-cli") +MODEL=os.environ.get("PK_MODEL", os.path.expanduser("~/parakeet-models/tdt_ctc-110m-q8_0.gguf")) +DECODER=os.environ.get("PK_DECODER","tdt"); PORT=int(os.environ.get("PK_PORT","8178")) +ENV=dict(os.environ); ENV["PATH"]="/usr/local/cuda/bin:"+ENV.get("PATH",""); ENV["LD_LIBRARY_PATH"]="/usr/local/cuda/lib64:"+ENV.get("LD_LIBRARY_PATH","") +_lock=threading.Lock() +proc=subprocess.Popen([BIN,"serve","--model",MODEL,"--decoder",DECODER],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True,bufsize=1,env=ENV) +ready=threading.Event() +def _drain(): + for ln in proc.stderr: + if "[serve] ready" in ln: ready.set() +threading.Thread(target=_drain,daemon=True).start() +ready.wait(timeout=120) +sys.stderr.write(f"[shim] serve ready={ready.is_set()} pid={proc.pid} alive={proc.poll() is None}\n"); sys.stderr.flush() +def _xc(wav_path): + with _lock: + proc.stdin.write(wav_path+"\n"); proc.stdin.flush() + return (proc.stdout.readline() or "").strip() +try: + w=sorted(glob.glob(os.path.expanduser("~/parakeet-eval/wav/*.wav")))[0]; _xc(w); sys.stderr.write("[shim] warmed\n"); sys.stderr.flush() +except Exception as e: sys.stderr.write(f"[shim] warmup skip: {e}\n") +def transcribe(b): + with tempfile.TemporaryDirectory() as d: + raw=os.path.join(d,"in.wav"); open(raw,"wb").write(b); w16=os.path.join(d,"16k.wav") + r=subprocess.run(["sox",raw,"-r","16000","-c","1","-b","16",w16],capture_output=True) + return _xc(w16 if (r.returncode==0 and os.path.exists(w16)) else raw) +class H(http.server.BaseHTTPRequestHandler): + protocol_version="HTTP/1.1" + def _j(self,o,c=200): + bb=json.dumps(o).encode(); self.send_response(c); self.send_header("Content-Type","application/json"); self.send_header("Content-Length",str(len(bb))); self.end_headers(); self.wfile.write(bb) + def do_GET(self): self._j({"status":"ok","engine":"parakeet-resident","model":os.path.basename(MODEL),"alive":proc.poll() is None}) + def do_POST(self): + if self.path.split("?")[0] not in ("/inference","/v1/audio/transcriptions"): self._j({"error":"not found"},404); return + fs=cgi.FieldStorage(fp=self.rfile,headers=self.headers,environ={"REQUEST_METHOD":"POST","CONTENT_TYPE":self.headers.get("Content-Type",""),"CONTENT_LENGTH":self.headers.get("Content-Length","0")}) + data=None + for k in ("file","audio_file","audio"): + if k in fs and getattr(fs[k],"file",None): data=fs[k].file.read(); break + if not data: self._j({"error":"no file field"},400); return + try: self._j({"text":transcribe(data)}) + except Exception as e: self._j({"error":str(e)},500) + def log_message(self,*a): pass +socketserver.TCPServer.allow_reuse_address=True +with socketserver.ThreadingTCPServer(("127.0.0.1",PORT),H) as s: + print(f"parakeet-resident-shim :{PORT} model={os.path.basename(MODEL)}",flush=True); s.serve_forever() diff --git a/deploy/parakeet-stt/serve-mode.patch b/deploy/parakeet-stt/serve-mode.patch new file mode 100644 index 0000000..ec7de5c --- /dev/null +++ b/deploy/parakeet-stt/serve-mode.patch @@ -0,0 +1,61 @@ +diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp +index ed7d017..5783773 100644 +--- a/examples/cli/main.cpp ++++ b/examples/cli/main.cpp +@@ -18,6 +18,7 @@ + #include + #include + #include ++#include + + static int cmd_info(const char* path) { + pk::ModelLoader ml; +@@ -647,6 +648,39 @@ static int run_and_shutdown(int (*fn)(int, char**), int argc, char** argv) { + return rc; + } + ++static int cmd_serve(int argc, char** argv) { ++ std::string model, decoder_str; ++ int threads = 0; ++ for (int i = 0; i < argc; ++i) { ++ if (std::strcmp(argv[i], "--model") == 0 && i + 1 < argc) model = argv[++i]; ++ else if (std::strcmp(argv[i], "--decoder") == 0 && i + 1 < argc) decoder_str = argv[++i]; ++ else if (std::strcmp(argv[i], "--threads") == 0 && i + 1 < argc) threads = std::atoi(argv[++i]); ++ } ++ if (model.empty()) { std::fprintf(stderr, "usage: parakeet-cli serve --model [--decoder ctc|tdt] [--threads N]\n"); return 2; } ++ if (threads > 0) pk::set_num_threads(threads); ++ pk::Decoder dec = pk::Decoder::kDefault; ++ if (decoder_str == "ctc") dec = pk::Decoder::kCTC; ++ else if (decoder_str == "tdt") dec = pk::Decoder::kTDT; ++ std::unique_ptr m = pk::Model::load(model); ++ if (!m) { std::fprintf(stderr, "parakeet-cli serve: failed to load model %s\n", model.c_str()); return 1; } ++ std::fprintf(stderr, "[serve] ready\n"); std::fflush(stderr); ++ std::string line; ++ while (std::getline(std::cin, line)) { ++ if (!line.empty() && line.back() == '\r') line.pop_back(); ++ std::string text; ++ if (!line.empty()) { ++ try { ++ pk::Audio audio; ++ if (pk::load_audio_16k_mono(line, audio)) text = m->transcribe_pcm(audio.samples, 16000, dec); ++ else std::fprintf(stderr, "[serve] failed to load audio %s\n", line.c_str()); ++ } catch (const std::exception& e) { std::fprintf(stderr, "[serve] transcribe failed: %s\n", e.what()); } ++ } ++ for (char& c : text) if (c == '\n' || c == '\r') c = ' '; ++ std::printf("%s\n", text.c_str()); std::fflush(stdout); ++ } ++ return 0; ++} ++ + int main(int argc, char** argv) { + if (argc >= 3 && std::strcmp(argv[1], "info") == 0) + return run_and_shutdown([](int, char** a) { return cmd_info(a[0]); }, 1, argv + 2); +@@ -656,6 +690,8 @@ int main(int argc, char** argv) { + return run_and_shutdown(cmd_quantize, argc - 2, argv + 2); + if (argc >= 2 && std::strcmp(argv[1], "bench") == 0) + return run_and_shutdown(cmd_bench, argc - 2, argv + 2); ++ if (argc >= 2 && std::strcmp(argv[1], "serve") == 0) ++ return run_and_shutdown(cmd_serve, argc - 2, argv + 2); + std::fprintf(stderr, + "usage:\n" + " parakeet-cli info \n" diff --git a/docs/stt-parakeet-evaluation.md b/docs/stt-parakeet-evaluation.md new file mode 100644 index 0000000..4882730 --- /dev/null +++ b/docs/stt-parakeet-evaluation.md @@ -0,0 +1,100 @@ +# STT: whisper.cpp → parakeet.cpp evaluation and swap + +This documents replacing the speech-to-text engine in the deployed voice pipeline +(`genie-claw` / GeniePod on the Jetson Orin Nano Super) from **whisper.cpp +(`ggml-small`)** to **parakeet.cpp (`tdt_ctc-110m`)**, and the head-to-head +comparison that motivated it. It closes the *"STT accuracy (WER) on real LyraT +captures vs clean reference"* open item in #2. + +## Summary + +On the target device, **parakeet `tdt_ctc-110m-q8_0` beats whisper `ggml-small` +on every axis** — accuracy, speed, and memory — and it is the *small* parakeet +model. + +| Metric (lower=better unless noted) | whisper `ggml-small` (original) | parakeet `tdt_ctc-110m-q8` (current) | +|---|---|---| +| **WER**, 50 clean clips (Whisper-normalized) | 1.02 % | **0.82 %** | +| **RTFx** amortized (higher=better) | 19× | **116×** | +| RTFx naive, incl. per-file model load | 5.1× | 13.8× | +| **Per-call latency**, resident server | ~500 ms (config-stated) | **~170 ms** (measured 168–189 ms) | +| **Peak RAM** Δ over baseline | +726 MB | **+410 MB** | +| Model on disk | 466 MB | **170 MB** | +| Real LyraT capture | correct | correct | + +Net: **more accurate, ~6× faster compute, ~3× lower per-call latency, ~300 MB +leaner.** + +## Test setup + +- **Device:** Jetson Orin Nano Super 8 GB · JetPack 6.2 (L4T 36.4.7) · CUDA 12.6 · + GPU `CUDA0` (Orin, compute 8.7). Unified 7.6 GB. Both engines measured with the + genie LLM/STT services stopped so the GPU was uncontended and conditions were + identical. +- **Eval set:** first 50 LibriSpeech `test-clean` clips (≈6.3 min), references from + the corpus. WER via the Whisper normalizer (Open ASR Leaderboard convention, + `whisper-normalizer`) + `jiwer`, applied identically to both engines. +- **whisper:** `/opt/geniepod/models/ggml-small.bin` via `whisper-cli` and the + long-running `whisper-server` (`POST /inference`, the deployed integration). +- **parakeet:** `tdt_ctc-110m-q8_0.gguf` (177,796,224 bytes), parakeet.cpp + `v0.1.1`, decoder `tdt`. + +### Real-capture validation + +Live ESP32-LyraT mic → I2S2 → ADMAIF1 → `plughw:APE,0` (24 kHz, per #2) → resampled +to 16 kHz mono → parakeet. Three live utterances transcribed correctly, e.g. +*"Hello, this is … okay, thank you."* and *"I'm a software engineer, can you +check me?"* (minor proper-noun spelling only). The clean-set WER is therefore a +ceiling; a labeled real-capture set is still needed to quantify the noisy-24 kHz +number at scale (tracked back into #2). + +## What we changed + +parakeet.cpp `v0.1.1` is **CLI-only** and reloads the 170 MB model on every +invocation (~1.3 s), which cannot match whisper-*server*'s resident ~500 ms. The +deployed integration point is whisper-server's `POST /inference` on +`127.0.0.1:8178` (genie-core calls it; config `whisper_port = 8178`). So: + +1. **Resident `serve` mode** — patched parakeet.cpp's CLI to add a `serve` + subcommand: load the model **once**, then read wav paths on stdin and emit one + transcript per line. See [`deploy/parakeet-stt/serve-mode.patch`](../deploy/parakeet-stt/serve-mode.patch). +2. **Drop-in HTTP shim** — [`deploy/parakeet-stt/parakeet-stt-server.py`](../deploy/parakeet-stt/parakeet-stt-server.py) + spawns `parakeet-cli serve` once and serves the **same** `POST /inference` → + `{"text": ...}` contract on `:8178` (sox-resamples input to 16 kHz mono). + Measured **~170 ms/call resident**. +3. **systemd service** — [`deploy/parakeet-stt/genie-parakeet.service`](../deploy/parakeet-stt/genie-parakeet.service) + runs the shim (`Restart=always`, starts on boot, after `genie-ai-runtime`). +4. **Swap** — `systemctl disable --now genie-whisper` + `enable --now + genie-parakeet`. **genie-core is unchanged** (it still posts to `:8178`; + parakeet now answers). Fully reversible. + +See [`deploy/parakeet-stt/README.md`](../deploy/parakeet-stt/README.md) for the +exact build + install + revert commands. + +## Caveats + +- Clean LibriSpeech subset — both engines score very low; the real win on noisy + 24 kHz LyraT captures is not yet quantified at scale (only the single live + capture above). +- `tdt_ctc-110m` is the **lean** model. `tdt-0.6b-v2-q8` is more accurate + (~1.7 % on the leaderboard) but ~904 MB and heavier to load/run — swap it in via + the service's `PK_MODEL` env if accuracy outweighs footprint. +- Numbers are `q8_0` quantization on this exact device; treat as device-specific. + +## Build notes (Jetson, parakeet.cpp v0.1.1) + +- Build **natively** with `cmake` — **not** the `build-aarch64-linux-gnu.sh` + cross-compile script. `nvcc` must be on `PATH` (`/usr/local/cuda/bin`); pass + `-DPARAKEET_GGML_CUDA=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DCMAKE_CUDA_ARCHITECTURES=87`. +- Runtime needs `LD_LIBRARY_PATH=/usr/local/cuda/lib64`. +- parakeet requires **16 kHz mono** input (it rejects 48 kHz/stereo; the shim + resamples with `sox`). +- Prebuilt GGUF models: `huggingface.co/mudler/parakeet-cpp-gguf`. + +## Optional: streaming + +The cache-aware streaming model `realtime_eou_120m-v1` runs on this device +(`parakeet-cli transcribe --stream`) and produces incremental partials, +end-of-utterance (`[EOU]`) detection, and per-word timestamps — enabling +EOU-driven turn-taking instead of fixed-window recording. Validated but not wired +into the runtime; left as a follow-on.