Skip to content

Commit fc7e922

Browse files
authored
feat(ai): add Google embedContent embeddings (#62843)
1 parent 2a9a3ed commit fc7e922

11 files changed

Lines changed: 639 additions & 6 deletions

‎packages/ai/README.md‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -409,6 +409,31 @@ OpenAICompatible.configure({ provider: "nvidia", baseURL: "https://integrate.api
409409
Every host here uses OpenAI's `/embeddings` API; Cohere through its compatibility API
410410
(`https://api.cohere.ai/compatibility/v1`). Package entry points are `@opencode/ai/providers/<provider>/embeddings`.
411411

412+
Google embeds through `embedContent`, which also takes images, audio, video and PDFs on `gemini-embedding-2`. Each
413+
value is its own content and gets its own vector (several parts in one content would be averaged into one). `dimensions`
414+
is `outputDimensionality`, `inputType` becomes `taskType` `RETRIEVAL_QUERY` or `RETRIEVAL_DOCUMENT`, and `taskType` or
415+
`title` in `providerOptions` replace or add to it. `gemini-embedding-2` ignores task types; Google recommends writing the
416+
task into the text instead (`task: search result | query: …`, `title: none | text: …`).
417+
418+
```ts
419+
import { Google, GoogleVertex } from "@opencode/ai/providers"
420+
421+
const response = await ai.embedding.generate({
422+
model: Google.configure().embedding("gemini-embedding-2"),
423+
values: [await ai.file("./receipt.png"), "A receipt for two coffees."],
424+
dimensions: 768,
425+
})
426+
427+
GoogleVertex.configure({ project, location: "global" }).embedding("gemini-embedding-2")
428+
```
429+
430+
The Gemini API sends up to 100 values per `batchEmbedContents` request. Vertex `embedContent` takes one value per
431+
request, so a call sends one request per value, four at a time, and needs project credentials: Vertex Express Mode API
432+
keys can't call `embedContent`. `gemini-embedding-2` runs in the `global`, `us` and `eu` Vertex locations. When a call
433+
is split into several requests, the vectors stay in value order, `usage.inputTokens` is their sum, and each request's
434+
metadata is listed in order under `providerMetadata.<provider>.responses`. Google has no `/embeddings` entry point yet;
435+
use `embeddingModel` from `@opencode/ai/providers/google` or `@opencode/ai/providers/google-vertex`.
436+
412437
## Alibaba Cloud Model Studio
413438

414439
`Alibaba` provides standard Model Studio inference. Configure a region explicitly, then select
Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
import { Effect, Schema } from "effect"
2+
import { EmbeddingResponse, EmbeddingUsage, type EmbeddingRequest } from "../embedding.js"
3+
import { EmbeddingRoute } from "../route/embedding.js"
4+
import { Endpoint } from "../route/endpoint.js"
5+
import { MediaProtocol } from "../route/media-protocol.js"
6+
import { mergeJsonRecords, type AIError, type OpenString } from "../schema/index.js"
7+
import { GeminiGenerateContent } from "./utils/gemini-generate-content.js"
8+
9+
const ID = "google-embed-content"
10+
const NAME = "Google embedContent"
11+
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
12+
/** `batchEmbedContents` takes at most 100 requests. */
13+
export const MAX_BATCH = 100
14+
15+
export type TaskType = OpenString<
16+
| "RETRIEVAL_QUERY"
17+
| "RETRIEVAL_DOCUMENT"
18+
| "SEMANTIC_SIMILARITY"
19+
| "CLASSIFICATION"
20+
| "CLUSTERING"
21+
| "QUESTION_ANSWERING"
22+
| "FACT_VERIFICATION"
23+
| "CODE_RETRIEVAL_QUERY"
24+
>
25+
26+
/**
27+
* Per-value embedding fields, sent with each value. `taskType` replaces the one `inputType` maps to; Gemini Embedding 2
28+
* ignores task types and documents task prefixes in the text instead (`task: search result | query: …`).
29+
*/
30+
export interface OptionsInput {
31+
readonly [key: string]: unknown
32+
readonly taskType?: TaskType
33+
readonly title?: string
34+
}
35+
36+
/** Vertex `embedContentConfig` fields; Vertex alone takes `autoTruncate`, `documentOcr` and `audioTrackExtraction`. */
37+
export interface VertexOptionsInput extends OptionsInput {
38+
readonly autoTruncate?: boolean
39+
readonly documentOcr?: boolean
40+
readonly audioTrackExtraction?: boolean
41+
}
42+
43+
const Values = Schema.Struct({ values: Schema.Array(Schema.Number) })
44+
45+
const UsageMetadata = Schema.StructWithRest(Schema.Struct({ promptTokenCount: Schema.optional(Schema.Number) }), [
46+
Schema.Record(Schema.String, Schema.Unknown),
47+
])
48+
49+
const BatchResponse = Schema.Struct({
50+
embeddings: Schema.Array(Values),
51+
usageMetadata: Schema.optional(UsageMetadata),
52+
})
53+
54+
const SingleResponse = Schema.Struct({
55+
embedding: Values,
56+
usageMetadata: Schema.optional(UsageMetadata),
57+
truncated: Schema.optional(Schema.Boolean),
58+
})
59+
60+
const TASK_TYPES = { query: "RETRIEVAL_QUERY", document: "RETRIEVAL_DOCUMENT" } as const
61+
62+
// Each value is its own content: several parts in one content embed into one aggregated vector.
63+
const content = (value: EmbeddingRequest["values"][number]): Effect.Effect<{ readonly parts: [unknown] }, AIError> =>
64+
typeof value === "string"
65+
? Effect.succeed({ parts: [{ text: value }] })
66+
: GeminiGenerateContent.mediaPart(NAME, value).pipe(Effect.map((part) => ({ parts: [part] })))
67+
68+
const config = (request: EmbeddingRequest) => ({
69+
...(request.inputType === undefined ? {} : { taskType: TASK_TYPES[request.inputType] }),
70+
...(request.dimensions === undefined ? {} : { outputDimensionality: request.dimensions }),
71+
...request.providerOptions,
72+
})
73+
74+
const identity = MediaProtocol.identity({ id: ID, name: NAME, provider: "google" })
75+
const decodeBatch = identity.decodeJson(BatchResponse)
76+
const decodeSingle = identity.decodeJson(SingleResponse)
77+
78+
const usage = (metadata: Schema.Schema.Type<typeof UsageMetadata> | undefined) =>
79+
metadata?.promptTokenCount === undefined ? undefined : new EmbeddingUsage({ inputTokens: metadata.promptTokenCount })
80+
81+
/** Gemini API `batchEmbedContents`: one request per value, so every value gets its own vector. */
82+
export const protocol = EmbeddingRoute.protocol({
83+
id: ID,
84+
name: NAME,
85+
body: {
86+
from: Effect.fn("GoogleEmbedContent.fromRequest")(function* (request: EmbeddingRequest) {
87+
const contents = yield* Effect.forEach(request.values, content)
88+
const fields = config(request)
89+
return (
90+
mergeJsonRecords(
91+
{ requests: contents.map((item) => ({ model: `models/${request.model.id}`, content: item, ...fields })) },
92+
request.http?.body,
93+
) ?? {}
94+
)
95+
}),
96+
},
97+
response: {
98+
decode: Effect.fn("GoogleEmbedContent.decodeResponse")(function* (response, context) {
99+
const output = yield* decodeBatch(response)
100+
return new EmbeddingResponse({
101+
embeddings: output.value.embeddings.map((item) => item.values),
102+
usage: usage(output.value.usageMetadata),
103+
providerMetadata: {
104+
[context.providerMetadataKey]:
105+
output.value.usageMetadata === undefined ? {} : { usageMetadata: output.value.usageMetadata },
106+
},
107+
})
108+
}),
109+
},
110+
})
111+
112+
/**
113+
* Vertex `embedContent`: one content per request, with options under `embedContentConfig`. Routes using it set
114+
* `maxValues: 1`.
115+
*/
116+
export const vertexProtocol = EmbeddingRoute.protocol({
117+
id: "google-vertex-embed-content",
118+
name: "Google Vertex embedContent",
119+
body: {
120+
from: Effect.fn("GoogleEmbedContent.vertexFromRequest")(function* (request: EmbeddingRequest) {
121+
const contents = yield* Effect.forEach(request.values, content)
122+
const fields = config(request)
123+
return (
124+
mergeJsonRecords(
125+
{
126+
content: contents[0],
127+
...(Object.keys(fields).length === 0 ? {} : { embedContentConfig: fields }),
128+
},
129+
request.http?.body,
130+
) ?? {}
131+
)
132+
}),
133+
},
134+
response: {
135+
decode: Effect.fn("GoogleEmbedContent.vertexDecodeResponse")(function* (response, context) {
136+
const output = yield* decodeSingle(response)
137+
return new EmbeddingResponse({
138+
embeddings: [output.value.embedding.values],
139+
usage: usage(output.value.usageMetadata),
140+
providerMetadata: {
141+
[context.providerMetadataKey]: {
142+
...(output.value.usageMetadata === undefined ? {} : { usageMetadata: output.value.usageMetadata }),
143+
...(output.value.truncated === undefined ? {} : { truncated: output.value.truncated }),
144+
},
145+
},
146+
})
147+
}),
148+
},
149+
})
150+
151+
export const route = EmbeddingRoute.make({
152+
id: ID,
153+
provider: "google",
154+
protocol,
155+
endpoint: Endpoint.path(({ request }) => `/models/${request.model.id}:batchEmbedContents`, {
156+
baseURL: DEFAULT_BASE_URL,
157+
}),
158+
maxValues: MAX_BATCH,
159+
})
160+
161+
export const GoogleEmbedContent = {
162+
protocol,
163+
vertexProtocol,
164+
route,
165+
} as const

‎packages/ai/src/protocols/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ export * as BedrockConverse from "./bedrock-converse.js"
33
export * as CloudflareRunDecisions from "./cloudflare-run-decisions.js"
44
export * as CohereChat from "./cohere-chat.js"
55
export * as Gemini from "./gemini.js"
6+
export * as GoogleEmbedContent from "./google-embed-content.js"
67
export * as GoogleInteractions from "./google-interactions.js"
78
export * as MistralChat from "./mistral-chat.js"
89
export * as OpenAIChat from "./openai-chat.js"

‎packages/ai/src/providers/google-vertex.ts‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
import { Effect } from "effect"
22
import type { ProviderPackage } from "../provider-package.js"
33
import { Gemini } from "../protocols/gemini.js"
4+
import { GoogleEmbedContent, type VertexOptionsInput } from "../protocols/google-embed-content.js"
45
import { GoogleImages } from "../protocols/google-images.js"
56
import { ProviderShared } from "../protocols/shared.js"
67
import { Auth } from "../route/auth.js"
78
import { Route } from "../route/client.js"
9+
import { EmbeddingRoute } from "../route/embedding.js"
810
import { Endpoint } from "../route/endpoint.js"
911
import { MediaRoute } from "../route/media.js"
1012
import {
@@ -82,6 +84,18 @@ const route = Route.make({
8284

8385
export const routes = [route]
8486

87+
export type { VertexOptionsInput as GoogleVertexEmbeddingOptions } from "../protocols/google-embed-content.js"
88+
89+
const embeddingRoute = EmbeddingRoute.make({
90+
id: "google-vertex-embed-content",
91+
provider: id,
92+
providerMetadataKey: "vertex",
93+
protocol: GoogleEmbedContent.vertexProtocol,
94+
endpoint: Endpoint.path(({ request }) => `/models/${request.model.id}:embedContent`),
95+
// Vertex embedContent embeds one content per request.
96+
maxValues: 1,
97+
})
98+
8599
const configuredRoute = (input: Config, modelID: string | ModelID) => {
86100
const {
87101
accessToken: _accessToken,
@@ -129,6 +143,18 @@ export const configure = (input: Config = {}) => {
129143
providerOptions,
130144
})
131145
},
146+
/** `gemini-embedding-2` through Vertex `embedContent`; it needs project credentials, not an Express Mode key. */
147+
embedding: (modelID: string | ModelID, providerOptions?: ProviderOptions) => {
148+
const target = connection(input, modelID)
149+
if (GoogleVertexShared.apiKey(input) !== undefined && input.baseURL === undefined)
150+
throw new ProviderConfigurationError({
151+
provider: id,
152+
message: "Google Vertex embedContent does not support Express Mode API keys; use project credentials",
153+
})
154+
return embeddingRoute
155+
.with({ endpoint: target.endpoint, auth: target.auth, headers: input.headers, http: input.http })
156+
.model<VertexOptionsInput>({ id: modelID, providerOptions })
157+
},
132158
configure,
133159
}
134160
}
@@ -143,6 +169,10 @@ export const imageModel = (modelID: string, settings?: Settings) => {
143169
const input = config(settings)
144170
return configure(input).image(modelID, input.providerOptions)
145171
}
172+
export const embeddingModel = (modelID: string, settings?: Settings) => {
173+
const input = config(settings)
174+
return configure(input).embedding(modelID, input.providerOptions)
175+
}
146176

147177
function config({ accessToken, apiKey, baseURL, body, headers, location, project, ...providerOptions }: Settings = {}) {
148178
if (apiKey !== undefined && accessToken !== undefined)

‎packages/ai/src/providers/google.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,18 @@ import { MediaRoute } from "../route/media.js"
44
import type { ProviderPackage } from "../provider-package.js"
55
import { ProviderID, type ModelID, type ProviderOptions, type RequestOptions } from "../schema/index.js"
66
import { Gemini } from "../protocols/gemini.js"
7+
import {
8+
GoogleEmbedContent,
9+
type OptionsInput as GoogleEmbeddingOptionsInput,
10+
} from "../protocols/google-embed-content.js"
711
import { GoogleInteractions } from "../protocols/google-interactions.js"
812
import { GoogleImages } from "../protocols/google-images.js"
913
import { GoogleSpeech } from "../protocols/google-speech.js"
1014
import { GoogleTranscription } from "../protocols/google-transcription.js"
1115
import { GoogleVideo } from "../protocols/google-video.js"
1216
import { toConfig } from "./settings.js"
1317

18+
export type { OptionsInput as GoogleEmbeddingOptions } from "../protocols/google-embed-content.js"
1419
export type { GoogleImageOptions } from "../protocols/google-images.js"
1520
export type { GoogleSpeechOptions } from "../protocols/google-speech.js"
1621
export type { GoogleTranscriptionOptions } from "../protocols/google-transcription.js"
@@ -52,6 +57,13 @@ export const configure = (input: Config = {}) => {
5257
const route = configuredRoute(input)
5358
const media = MediaRoute.deployment(input, auth(input))
5459
const gemini = (modelID: string | ModelID) => route.model<Gemini.ProviderOptionsInput>({ id: modelID })
60+
const embeddingRoute = GoogleEmbedContent.route.with({
61+
provider: id,
62+
endpoint: { baseURL: input.baseURL },
63+
auth: auth(input),
64+
headers: input.headers,
65+
http: input.http,
66+
})
5567
return {
5668
id,
5769
model: gemini,
@@ -63,6 +75,8 @@ export const configure = (input: Config = {}) => {
6375
video: (modelID: string | ModelID) => GoogleVideo.model({ ...media, id: modelID }),
6476
speech: (modelID: string | ModelID) => GoogleSpeech.model({ ...media, id: modelID }),
6577
transcription: (modelID: string | ModelID) => GoogleTranscription.model({ ...media, id: modelID }),
78+
embedding: (modelID: string | ModelID, providerOptions?: ProviderOptions) =>
79+
embeddingRoute.model<GoogleEmbeddingOptionsInput>({ id: modelID, providerOptions }),
6680
configure,
6781
}
6882
}
@@ -74,8 +88,13 @@ export const imageModel = (modelID: string, settings?: Settings) => {
7488
const input = toConfig(settings)
7589
return configure(input).image(modelID, input.providerOptions)
7690
}
91+
export const embeddingModel = (modelID: string, settings?: Settings) => {
92+
const input = toConfig(settings)
93+
return configure(input).embedding(modelID, input.providerOptions)
94+
}
7795

7896
export const image = provider.image
97+
export const embedding = provider.embedding
7998
export const gemini = provider.gemini
8099
export const interactions = provider.interactions
81100
export const video = provider.video

0 commit comments

Comments
 (0)