diff --git a/apps/desktop/src/pipeline/providers/transcription/whisper-prompt.ts b/apps/desktop/src/pipeline/providers/transcription/whisper-prompt.ts index af96ce6a..0831f3bc 100644 --- a/apps/desktop/src/pipeline/providers/transcription/whisper-prompt.ts +++ b/apps/desktop/src/pipeline/providers/transcription/whisper-prompt.ts @@ -59,6 +59,11 @@ export function sanitizeWhisperPrompt(prompt: string): string { } export interface BuildWhisperPromptOptions { + /** Candidate language codes passed to Whisper. Exactly one entry forces + * that language; in that mode target-app text is excluded because it can + * override the explicit language token. */ + languages?: readonly string[]; + /** Domain vocabulary / hotwords (proper nouns, jargon). Joined with ", ". * Placed at the START of the prompt: less critical for continuity than the * prior-transcript tail, and if byte truncation hits, the start gets dropped @@ -97,7 +102,8 @@ export function buildWhisperPrompt( if (vocab) parts.push(vocab); } - const source = opts.previousTranscription || opts.beforeText; + const beforeText = opts.languages?.length === 1 ? undefined : opts.beforeText; + const source = opts.previousTranscription || beforeText; if (source) { const words = source.trim().split(/\s+/).filter(Boolean); const n = opts.previousWordCount ?? DEFAULT_PREVIOUS_WORD_COUNT; diff --git a/apps/desktop/src/pipeline/providers/transcription/whisper-provider.ts b/apps/desktop/src/pipeline/providers/transcription/whisper-provider.ts index f59093ac..c8f3ea02 100644 --- a/apps/desktop/src/pipeline/providers/transcription/whisper-provider.ts +++ b/apps/desktop/src/pipeline/providers/transcription/whisper-provider.ts @@ -420,7 +420,11 @@ export class WhisperProvider implements TranscriptionEngine { return this.resourceMutex.runExclusive(async () => { this.assertNotDisposed(); const worker = await this.initializeWhisperResource(modelPath); + // One language forces it; multiple drives constrained auto-detection + // in the addon; empty/undefined means auto-detect. + const languages = localWhisperLanguages(context.languages, modelPath); const initialPrompt = this.generateInitialPrompt( + languages, context.vocabulary, context.aggregatedTranscription, context.accessibilityContext, @@ -429,9 +433,7 @@ export class WhisperProvider implements TranscriptionEngine { const result = await worker.exec("transcribeAudio", [ aggregatedAudio, { - // One language forces it; multiple drives constrained auto-detection - // in the addon; empty/undefined means auto-detect. - languages: localWhisperLanguages(context.languages, modelPath), + languages, initial_prompt: initialPrompt, suppress_blank: true, suppress_non_speech_tokens: true, @@ -474,11 +476,13 @@ export class WhisperProvider implements TranscriptionEngine { } private generateInitialPrompt( + languages?: readonly string[], vocabulary?: readonly string[], aggregatedTranscription?: string, accessibilityContext?: TranscribeContext["accessibilityContext"], ): string { const prompt = buildWhisperPrompt({ + languages, vocabulary, previousTranscription: aggregatedTranscription, beforeText: diff --git a/apps/desktop/tests/pipeline/whisper-prompt.test.ts b/apps/desktop/tests/pipeline/whisper-prompt.test.ts new file mode 100644 index 00000000..67c000f8 --- /dev/null +++ b/apps/desktop/tests/pipeline/whisper-prompt.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { buildWhisperPrompt } from "../../src/pipeline/providers/transcription/whisper-prompt"; + +describe("buildWhisperPrompt language conditioning", () => { + it("omits target-app text when exactly one language is forced", () => { + expect( + buildWhisperPrompt({ + languages: ["ru"], + beforeText: "ctrl esc to focus or unfocus Claude", + }), + ).toBeUndefined(); + }); + + it("keeps vocabulary and previous transcription for a forced language", () => { + expect( + buildWhisperPrompt({ + languages: ["ru"], + vocabulary: ["Amical", "Whisper"], + previousTranscription: "предыдущая русская фраза", + beforeText: "Write a message in English", + }), + ).toBe("Amical, Whisper. предыдущая русская фраза"); + }); + + it.each([ + ["auto-detection", undefined], + ["an empty language list", []], + ["constrained multi-language detection", ["en", "ru"]], + ])("keeps target-app text for %s", (_label, languages) => { + expect( + buildWhisperPrompt({ + languages, + beforeText: "Write a message in English", + }), + ).toBe("Write a message in English"); + }); +}); diff --git a/apps/desktop/tests/pipeline/whisper-provider-session.test.ts b/apps/desktop/tests/pipeline/whisper-provider-session.test.ts index 64a473b8..24904be2 100644 --- a/apps/desktop/tests/pipeline/whisper-provider-session.test.ts +++ b/apps/desktop/tests/pipeline/whisper-provider-session.test.ts @@ -1,4 +1,5 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { GetAccessibilityContextResult } from "@amical/types"; const workerMocks = vi.hoisted(() => ({ construct: vi.fn(), @@ -129,6 +130,47 @@ describe("WhisperProvider sessions", () => { return modelBySample; }; + it.each([ + { selected: ["ru"], decoded: ["ru"] }, + { selected: ["zh-Hans", "yue"], decoded: ["zh"] }, + ])( + "omits target-app text for decoder language $decoded", + async ({ selected, decoded }) => { + const { buildWhisperPrompt } = await vi.importActual< + typeof import("../../src/pipeline/providers/transcription/whisper-prompt") + >("../../src/pipeline/providers/transcription/whisper-prompt"); + processingMocks.buildWhisperPrompt.mockImplementationOnce( + buildWhisperPrompt, + ); + + const { provider } = createProvider(); + const session = provider.openSession({ sessionId: "pinned-language" }); + const context = { + languages: selected, + vocabulary: ["Amical"], + accessibilityContext: { + context: { + textSelection: { preSelectionText: "Write a message in English" }, + }, + } as unknown as GetAccessibilityContextResult, + }; + + await session.transcribe({ + audioData: new Float32Array([1]), + context, + }); + await session.flush(context); + + expect(workerMocks.exec).toHaveBeenCalledWith("transcribeAudio", [ + expect.any(Float32Array), + expect.objectContaining({ + languages: decoded, + initial_prompt: "Amical", + }), + ]); + }, + ); + it("isolates buffered audio and cancellation between sessions", async () => { const { provider } = createProvider(); const first = provider.openSession({ @@ -693,6 +735,7 @@ describe("WhisperProvider sessions", () => { expect(Array.from(rawAudio as Float32Array)).toEqual([0.5, 0.25]); expect(vadProbabilities).toEqual([0.8, 0.6]); expect(processingMocks.buildWhisperPrompt).toHaveBeenCalledWith({ + languages: ["en", "hi"], vocabulary: ["Amical"], previousTranscription: "prior words", beforeText: undefined,