Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,11 @@ export function sanitizeWhisperPrompt(prompt: string): string {
}

export interface BuildWhisperPromptOptions {
/** Candidate language codes passed to Whisper. Exactly one entry forces
* that language; in that mode target-app text is excluded because it can
* override the explicit language token. */
languages?: readonly string[];

/** Domain vocabulary / hotwords (proper nouns, jargon). Joined with ", ".
* Placed at the START of the prompt: less critical for continuity than the
* prior-transcript tail, and if byte truncation hits, the start gets dropped
Expand Down Expand Up @@ -97,7 +102,8 @@ export function buildWhisperPrompt(
if (vocab) parts.push(vocab);
}

const source = opts.previousTranscription || opts.beforeText;
const beforeText = opts.languages?.length === 1 ? undefined : opts.beforeText;
const source = opts.previousTranscription || beforeText;
if (source) {
const words = source.trim().split(/\s+/).filter(Boolean);
const n = opts.previousWordCount ?? DEFAULT_PREVIOUS_WORD_COUNT;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -420,7 +420,11 @@ export class WhisperProvider implements TranscriptionEngine {
return this.resourceMutex.runExclusive(async () => {
this.assertNotDisposed();
const worker = await this.initializeWhisperResource(modelPath);
// One language forces it; multiple drives constrained auto-detection
// in the addon; empty/undefined means auto-detect.
const languages = localWhisperLanguages(context.languages, modelPath);
const initialPrompt = this.generateInitialPrompt(
languages,
context.vocabulary,
context.aggregatedTranscription,
context.accessibilityContext,
Expand All @@ -429,9 +433,7 @@ export class WhisperProvider implements TranscriptionEngine {
const result = await worker.exec<TranscriptionOutput>("transcribeAudio", [
aggregatedAudio,
{
// One language forces it; multiple drives constrained auto-detection
// in the addon; empty/undefined means auto-detect.
languages: localWhisperLanguages(context.languages, modelPath),
languages,
initial_prompt: initialPrompt,
suppress_blank: true,
suppress_non_speech_tokens: true,
Expand Down Expand Up @@ -474,11 +476,13 @@ export class WhisperProvider implements TranscriptionEngine {
}

private generateInitialPrompt(
languages?: readonly string[],
vocabulary?: readonly string[],
aggregatedTranscription?: string,
accessibilityContext?: TranscribeContext["accessibilityContext"],
): string {
const prompt = buildWhisperPrompt({
languages,
vocabulary,
previousTranscription: aggregatedTranscription,
beforeText:
Expand Down
37 changes: 37 additions & 0 deletions apps/desktop/tests/pipeline/whisper-prompt.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import { describe, expect, it } from "vitest";
import { buildWhisperPrompt } from "../../src/pipeline/providers/transcription/whisper-prompt";

describe("buildWhisperPrompt language conditioning", () => {
it("omits target-app text when exactly one language is forced", () => {
expect(
buildWhisperPrompt({
languages: ["ru"],
beforeText: "ctrl esc to focus or unfocus Claude",
}),
).toBeUndefined();
});

it("keeps vocabulary and previous transcription for a forced language", () => {
expect(
buildWhisperPrompt({
languages: ["ru"],
vocabulary: ["Amical", "Whisper"],
previousTranscription: "предыдущая русская фраза",
beforeText: "Write a message in English",
}),
).toBe("Amical, Whisper. предыдущая русская фраза");
});

it.each([
["auto-detection", undefined],
["an empty language list", []],
["constrained multi-language detection", ["en", "ru"]],
])("keeps target-app text for %s", (_label, languages) => {
expect(
buildWhisperPrompt({
languages,
beforeText: "Write a message in English",
}),
).toBe("Write a message in English");
});
});
43 changes: 43 additions & 0 deletions apps/desktop/tests/pipeline/whisper-provider-session.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { GetAccessibilityContextResult } from "@amical/types";

const workerMocks = vi.hoisted(() => ({
construct: vi.fn(),
Expand Down Expand Up @@ -129,6 +130,47 @@ describe("WhisperProvider sessions", () => {
return modelBySample;
};

it.each([
{ selected: ["ru"], decoded: ["ru"] },
{ selected: ["zh-Hans", "yue"], decoded: ["zh"] },
])(
"omits target-app text for decoder language $decoded",
async ({ selected, decoded }) => {
const { buildWhisperPrompt } = await vi.importActual<
typeof import("../../src/pipeline/providers/transcription/whisper-prompt")
>("../../src/pipeline/providers/transcription/whisper-prompt");
processingMocks.buildWhisperPrompt.mockImplementationOnce(
buildWhisperPrompt,
);

const { provider } = createProvider();
const session = provider.openSession({ sessionId: "pinned-language" });
const context = {
languages: selected,
vocabulary: ["Amical"],
accessibilityContext: {
context: {
textSelection: { preSelectionText: "Write a message in English" },
},
} as unknown as GetAccessibilityContextResult,
};

await session.transcribe({
audioData: new Float32Array([1]),
context,
});
await session.flush(context);

expect(workerMocks.exec).toHaveBeenCalledWith("transcribeAudio", [
expect.any(Float32Array),
expect.objectContaining({
languages: decoded,
initial_prompt: "Amical",
}),
]);
},
);

it("isolates buffered audio and cancellation between sessions", async () => {
const { provider } = createProvider();
const first = provider.openSession({
Expand Down Expand Up @@ -693,6 +735,7 @@ describe("WhisperProvider sessions", () => {
expect(Array.from(rawAudio as Float32Array)).toEqual([0.5, 0.25]);
expect(vadProbabilities).toEqual([0.8, 0.6]);
expect(processingMocks.buildWhisperPrompt).toHaveBeenCalledWith({
languages: ["en", "hi"],
vocabulary: ["Amical"],
previousTranscription: "prior words",
beforeText: undefined,
Expand Down
Loading