From 56ffeb7bb64d987653230c1a01f7012e48504921 Mon Sep 17 00:00:00 2001 From: Florian Meyer Date: Fri, 7 Aug 2026 11:31:18 +0200 Subject: [PATCH] fix(desktop): honor pinned Whisper language --- .../providers/transcription/whisper-prompt.ts | 8 +++- .../transcription/whisper-provider.ts | 3 ++ .../tests/pipeline/whisper-prompt.test.ts | 37 +++++++++++++++ .../tests/pipeline/whisper-provider.test.ts | 47 +++++++++++++++++++ 4 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 apps/desktop/tests/pipeline/whisper-prompt.test.ts create mode 100644 apps/desktop/tests/pipeline/whisper-provider.test.ts diff --git a/apps/desktop/src/pipeline/providers/transcription/whisper-prompt.ts b/apps/desktop/src/pipeline/providers/transcription/whisper-prompt.ts index 76e9f109..4ab80586 100644 --- a/apps/desktop/src/pipeline/providers/transcription/whisper-prompt.ts +++ b/apps/desktop/src/pipeline/providers/transcription/whisper-prompt.ts @@ -59,6 +59,11 @@ export function sanitizeWhisperPrompt(prompt: string): string { } export interface BuildWhisperPromptOptions { + /** Candidate language codes passed to Whisper. Exactly one entry forces + * that language; in that mode target-app text is excluded because it can + * override the explicit language token. */ + languages?: readonly string[]; + /** Domain vocabulary / hotwords (proper nouns, jargon). Joined with ", ". * Placed at the START of the prompt: less critical for continuity than the * prior-transcript tail, and if byte truncation hits, the start gets dropped @@ -97,7 +102,8 @@ export function buildWhisperPrompt( if (vocab) parts.push(vocab); } - const source = opts.previousTranscription || opts.beforeText; + const beforeText = opts.languages?.length === 1 ? undefined : opts.beforeText; + const source = opts.previousTranscription || beforeText; if (source) { const words = source.trim().split(/\s+/).filter(Boolean); const n = opts.previousWordCount ?? DEFAULT_PREVIOUS_WORD_COUNT; diff --git a/apps/desktop/src/pipeline/providers/transcription/whisper-provider.ts b/apps/desktop/src/pipeline/providers/transcription/whisper-provider.ts index c69e3427..895e734f 100644 --- a/apps/desktop/src/pipeline/providers/transcription/whisper-provider.ts +++ b/apps/desktop/src/pipeline/providers/transcription/whisper-provider.ts @@ -185,6 +185,7 @@ export class WhisperProvider implements TranscriptionProvider { // Generate initial prompt from recent context only (align with cloud) const initialPrompt = this.generateInitialPrompt( + languages, context.vocabulary, aggregatedTranscription, context.accessibilityContext, @@ -292,11 +293,13 @@ export class WhisperProvider implements TranscriptionProvider { } private generateInitialPrompt( + languages?: readonly string[], vocabulary?: readonly string[], aggregatedTranscription?: string, accessibilityContext?: TranscribeContext["accessibilityContext"], ): string { const prompt = buildWhisperPrompt({ + languages, vocabulary, previousTranscription: aggregatedTranscription, beforeText: diff --git a/apps/desktop/tests/pipeline/whisper-prompt.test.ts b/apps/desktop/tests/pipeline/whisper-prompt.test.ts new file mode 100644 index 00000000..67c000f8 --- /dev/null +++ b/apps/desktop/tests/pipeline/whisper-prompt.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { buildWhisperPrompt } from "../../src/pipeline/providers/transcription/whisper-prompt"; + +describe("buildWhisperPrompt language conditioning", () => { + it("omits target-app text when exactly one language is forced", () => { + expect( + buildWhisperPrompt({ + languages: ["ru"], + beforeText: "ctrl esc to focus or unfocus Claude", + }), + ).toBeUndefined(); + }); + + it("keeps vocabulary and previous transcription for a forced language", () => { + expect( + buildWhisperPrompt({ + languages: ["ru"], + vocabulary: ["Amical", "Whisper"], + previousTranscription: "предыдущая русская фраза", + beforeText: "Write a message in English", + }), + ).toBe("Amical, Whisper. предыдущая русская фраза"); + }); + + it.each([ + ["auto-detection", undefined], + ["an empty language list", []], + ["constrained multi-language detection", ["en", "ru"]], + ])("keeps target-app text for %s", (_label, languages) => { + expect( + buildWhisperPrompt({ + languages, + beforeText: "Write a message in English", + }), + ).toBe("Write a message in English"); + }); +}); diff --git a/apps/desktop/tests/pipeline/whisper-provider.test.ts b/apps/desktop/tests/pipeline/whisper-provider.test.ts new file mode 100644 index 00000000..8523ae4f --- /dev/null +++ b/apps/desktop/tests/pipeline/whisper-provider.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it, vi } from "vitest"; +import type { GetAccessibilityContextResult } from "@amical/types"; +import type { ModelService } from "../../src/services/model-service"; +import type { TranscribeContext } from "../../src/pipeline/core/pipeline-types"; +import { WhisperProvider } from "../../src/pipeline/providers/transcription/whisper-provider"; + +const targetAppContext = { + context: { + textSelection: { + preSelectionText: "ctrl esc to focus or unfocus Claude", + }, + }, +} as unknown as GetAccessibilityContextResult; + +describe("WhisperProvider initial prompt", () => { + it("does not send target-app text when one language is forced", async () => { + const provider = new WhisperProvider({} as ModelService); + const exec = vi.fn().mockResolvedValue({ text: "тест" }); + + vi.spyOn(provider, "initializeWhisper").mockResolvedValue(); + Object.assign(provider, { workerWrapper: { exec } }); + + const context: TranscribeContext = { + languages: ["ru"], + vocabulary: ["Amical"], + accessibilityContext: targetAppContext, + }; + + for (let frame = 0; frame < 3; frame++) { + await provider.transcribe({ + audioData: new Float32Array(512).fill(0.1), + speechProbability: 1, + context, + }); + } + await provider.flush(context); + + const transcribeCall = exec.mock.calls.find( + ([method]) => method === "transcribeAudio", + ); + expect(transcribeCall).toBeDefined(); + expect(transcribeCall?.[1][1]).toMatchObject({ + languages: ["ru"], + initial_prompt: "Amical", + }); + }); +});