Skip to content

Commit 0bd8cee

Browse files
authored
feat(ai): add Google Decisions (#63369)
1 parent 532d099 commit 0bd8cee

13 files changed

Lines changed: 495 additions & 2 deletions

File tree

‎packages/ai/package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@
7777
"./providers/google-vertex/messages": "./src/providers/google-vertex/messages.ts",
7878
"./providers/google-vertex/mistral": "./src/providers/google-vertex/mistral.ts",
7979
"./providers/google-vertex/responses": "./src/providers/google-vertex/responses.ts",
80+
"./providers/google/decisions": "./src/providers/google/decisions.ts",
8081
"./providers/google/embeddings": "./src/providers/google/embeddings.ts",
8182
"./providers/google/images": "./src/providers/google/images.ts",
8283
"./providers/google/interactions": "./src/providers/google/interactions.ts",
@@ -141,6 +142,7 @@
141142
"./protocols/cloudflare-run-decisions": "./src/protocols/cloudflare-run-decisions.ts",
142143
"./protocols/cloudflare-workers-ai-images": "./src/protocols/cloudflare-workers-ai-images.ts",
143144
"./protocols/deepgram-transcription": "./src/protocols/deepgram-transcription.ts",
145+
"./protocols/google-decisions": "./src/protocols/google-decisions.ts",
144146
"./protocols/google-images": "./src/protocols/google-images.ts",
145147
"./protocols/google-speech": "./src/protocols/google-speech.ts",
146148
"./protocols/google-transcription": "./src/protocols/google-transcription.ts",
Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
import { Effect, Schema } from "effect"
2+
import { DecisionResponse, DecisionUsage, type DecisionPart, type DecisionRequest } from "../decision.js"
3+
import type { Media } from "../media.js"
4+
import { DecisionRoute } from "../route/decision.js"
5+
import { Endpoint } from "../route/endpoint.js"
6+
import { MediaProtocol } from "../route/media-protocol.js"
7+
import {
8+
AIError,
9+
ProviderID,
10+
UnsupportedOperationError,
11+
mergeJsonRecords,
12+
type ProviderOptions,
13+
} from "../schema/index.js"
14+
import { encodeJson } from "../utils/json.js"
15+
import { JsonObject, knownString, ProviderShared } from "./shared.js"
16+
import { SystemOne } from "./system-one.js"
17+
import { MediaInput } from "./utils/media-input.js"
18+
19+
const ID = "google-decisions"
20+
const NAME = "Google Decisions"
21+
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
22+
23+
const Options = Schema.Struct({
24+
/** Token budget for every image and video input; `ultra_high` applies to images only. */
25+
mediaResolution: Schema.optional(knownString<"low" | "medium" | "high" | "ultra_high">()),
26+
})
27+
/** Known fields map to their wire names; anything else is sent as given. */
28+
export type OptionsInput = ProviderOptions & typeof Options.Encoded
29+
30+
const Usage = Schema.StructWithRest(
31+
Schema.Struct({
32+
total_input_tokens: Schema.optional(Schema.Number),
33+
total_output_tokens: Schema.optional(Schema.Number),
34+
total_tokens: Schema.optional(Schema.Number),
35+
}),
36+
[JsonObject],
37+
)
38+
const Response = Schema.Struct({
39+
answers: Schema.Record(Schema.String, SystemOne.Answer),
40+
usage: Schema.optional(Usage),
41+
})
42+
43+
const decodeOptions = ProviderShared.validateWith(Schema.decodeUnknownEffect(Options))
44+
45+
const fromRequest = Effect.fn("GoogleDecisions.fromRequest")(function* (request: DecisionRequest) {
46+
const options = yield* decodeOptions(request.providerOptions ?? {})
47+
const { mediaResolution: _, ...native } = request.providerOptions ?? {}
48+
return {
49+
...mergeJsonRecords(native, request.http?.body),
50+
model: request.model.id,
51+
input: separateText(
52+
yield* Effect.forEach(request.input, (part) => inputPart(request, part, options.mediaResolution)),
53+
),
54+
questions: {
55+
...(yield* SystemOne.encodeQuestions(request.questions)),
56+
// Google names System One's predicate criteria `true_description` and `false_description`.
57+
...Object.fromEntries(
58+
request.questions.flatMap((question) =>
59+
question.type === "predicate" && question.criteria !== undefined
60+
? [
61+
[
62+
question.name,
63+
{
64+
type: "noul",
65+
instructions: question.instructions,
66+
criteria: {
67+
...(question.criteria.true === undefined ? {} : { true_description: question.criteria.true }),
68+
...(question.criteria.false === undefined ? {} : { false_description: question.criteria.false }),
69+
},
70+
},
71+
],
72+
]
73+
: [],
74+
),
75+
),
76+
},
77+
}
78+
})
79+
80+
const inputPart = (
81+
request: DecisionRequest,
82+
part: DecisionPart,
83+
resolution: string | undefined,
84+
): Effect.Effect<Record<string, unknown>, AIError> => {
85+
if (part.type === "text") return Effect.succeed({ type: "text", text: part.text })
86+
// Google reads every text block as text; JSON is serialized so its keys are never parsed as content blocks.
87+
if (part.type === "json") return Effect.succeed({ type: "text", text: encodeJson(part.value) })
88+
return mediaPart(request, part.media, resolution)
89+
}
90+
91+
// Google joins adjacent text blocks with no separator, so each one after another starts on a new line.
92+
const separateText = (blocks: ReadonlyArray<Record<string, unknown>>) =>
93+
blocks.map((block, index) =>
94+
block.type === "text" && blocks[index - 1]?.type === "text" ? { ...block, text: `\n${block.text}` } : block,
95+
)
96+
97+
const mediaPart = (request: DecisionRequest, asset: Media.Asset, resolution: string | undefined) => {
98+
const type =
99+
asset.kind === "image" || asset.kind === "audio" || asset.kind === "video"
100+
? asset.kind
101+
: asset.mediaType.toLowerCase() === "application/pdf"
102+
? "document"
103+
: undefined
104+
if (type === undefined)
105+
return Effect.fail(
106+
new AIError({
107+
reason: new UnsupportedOperationError({
108+
operation: "decision.media",
109+
provider: request.model.provider,
110+
route: request.model.route.id,
111+
message: `${NAME} takes images, audio, video and PDFs; got ${asset.mediaType}`,
112+
}),
113+
}),
114+
)
115+
const block = {
116+
type,
117+
mime_type: asset.mediaType.toLowerCase(),
118+
// `ultra_high` is image-only, so video keeps Google's default for it.
119+
...(resolution !== undefined && (type === "image" || (type === "video" && resolution !== "ultra_high"))
120+
? { resolution }
121+
: {}),
122+
}
123+
const inline = asset.inline()
124+
if (inline) return Effect.succeed({ ...block, data: inline.base64 })
125+
// Google fetches URLs itself, so one that needs our download credentials can't be forwarded.
126+
if (asset.source.type === "url" && /^https:\/\//i.test(asset.source.url) && asset.headers === undefined)
127+
return Effect.succeed({ ...block, uri: asset.source.url })
128+
const file = MediaInput.refID(asset, ProviderID.make("google"))
129+
if (file !== undefined) return Effect.succeed({ ...block, uri: file })
130+
return Effect.fail(
131+
ProviderShared.invalidRequest(
132+
`${NAME} accepts inline media, public HTTPS URLs and Google Files API URIs; materialize ${asset.source.type} media first`,
133+
),
134+
)
135+
}
136+
137+
const decodeDocument = MediaProtocol.identity({ id: ID, name: NAME, provider: "google" }).decodeJson(Response)
138+
139+
export const protocol = DecisionRoute.protocol({
140+
id: ID,
141+
name: NAME,
142+
body: { from: fromRequest },
143+
response: {
144+
decode: Effect.fn("GoogleDecisions.decodeResponse")(function* (response, context) {
145+
const output = yield* decodeDocument(response)
146+
const data = output.value
147+
const answers = data.answers
148+
const names = context.request.questions.map((question) => question.name)
149+
if (Object.keys(answers).length !== names.length || !names.every((name) => Object.hasOwn(answers, name)))
150+
return yield* output.invalid(`${NAME} answers do not match the requested questions`)
151+
return new DecisionResponse({
152+
answers: context.request.questions.map((question) => SystemOne.toAnswer(question, answers[question.name])),
153+
usage:
154+
data.usage === undefined
155+
? undefined
156+
: new DecisionUsage({
157+
inputTokens: data.usage.total_input_tokens,
158+
outputTokens: data.usage.total_output_tokens,
159+
totalTokens: ProviderShared.totalTokens(
160+
data.usage.total_input_tokens,
161+
data.usage.total_output_tokens,
162+
data.usage.total_tokens,
163+
),
164+
}),
165+
providerMetadata:
166+
data.usage === undefined ? undefined : { [context.providerMetadataKey]: { usage: data.usage } },
167+
})
168+
}),
169+
},
170+
})
171+
172+
export const route = DecisionRoute.make({
173+
id: ID,
174+
provider: "google",
175+
protocol,
176+
endpoint: Endpoint.path("/decisions", { baseURL: DEFAULT_BASE_URL }),
177+
})
178+
179+
export * as GoogleDecisions from "./google-decisions.js"

‎packages/ai/src/protocols/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ export * as BedrockConverse from "./bedrock-converse.js"
33
export * as CloudflareRunDecisions from "./cloudflare-run-decisions.js"
44
export * as CohereChat from "./cohere-chat.js"
55
export * as Gemini from "./gemini.js"
6+
export * as GoogleDecisions from "./google-decisions.js"
67
export * as GoogleEmbedContent from "./google-embed-content.js"
78
export * as GoogleInteractions from "./google-interactions.js"
89
export * as GoogleVertexPredictEmbeddings from "./google-vertex-predict-embeddings.js"

‎packages/ai/src/protocols/system-one.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ const encodeQuestions = (questions: ReadonlyArray<DecisionQuestion>) =>
9494
if (new Set(keys).size !== keys.length)
9595
return Effect.fail(
9696
ProviderShared.invalidRequest(
97-
`System One choice options are strings; "${question.name}" has a boolean and a string with the same text`,
97+
`Decision choice options are sent as string keys; "${question.name}" has a boolean and a string with the same text`,
9898
),
9999
)
100100
return Effect.succeed([
@@ -294,4 +294,13 @@ export const route = DecisionRoute.make({
294294
endpoint: Endpoint.path("/systemone", { baseURL: DEFAULT_BASE_URL }),
295295
})
296296

297-
export const SystemOne = { Response, encodeBody, toResponse, protocol, route } as const
297+
export const SystemOne = {
298+
Answer,
299+
Response,
300+
encodeBody,
301+
encodeQuestions,
302+
toAnswer,
303+
toResponse,
304+
protocol,
305+
route,
306+
} as const

‎packages/ai/src/provider-error.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,12 @@ const CONTENT_POLICY_CODES = new Set([
131131
"refusal",
132132
"datainspectionfailed",
133133
"moderation_blocked",
134+
// Gemini Decisions rejects blocked input with these `error.code` strings on a 400.
135+
"safety",
136+
"prohibited_content",
137+
"blocklist",
138+
"image_safety",
139+
"content_blocked",
134140
])
135141
// OpenCode Zen replaces upstream codes outside its allow-list but keeps the original
136142
// as a `[code]` label at the start of the rewritten message.

‎packages/ai/src/providers/google.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import {
88
GoogleEmbedContent,
99
type OptionsInput as GoogleEmbeddingOptionsInput,
1010
} from "../protocols/google-embed-content.js"
11+
import { GoogleDecisions } from "../protocols/google-decisions.js"
1112
import { GoogleInteractions } from "../protocols/google-interactions.js"
1213
import { GoogleImages } from "../protocols/google-images.js"
1314
import { GoogleSpeech } from "../protocols/google-speech.js"
@@ -23,6 +24,7 @@ export type { GoogleVideoOptions } from "../protocols/google-video.js"
2324
export type GeminiOptionsInput = Gemini.OptionsInput
2425
export type GeminiProviderOptionsInput = Gemini.ProviderOptionsInput
2526
export type GoogleInteractionsOptionsInput = GoogleInteractions.OptionsInput
27+
export type GoogleDecisionsOptionsInput = GoogleDecisions.OptionsInput
2628

2729
export const id = ProviderID.make("google")
2830

@@ -77,6 +79,10 @@ export const configure = (input: Config = {}) => {
7779
transcription: (modelID: string | ModelID) => GoogleTranscription.model({ ...media, id: modelID }),
7880
embedding: (modelID: string | ModelID, providerOptions?: ProviderOptions) =>
7981
embeddingRoute.model<GoogleEmbeddingOptionsInput>({ id: modelID, providerOptions }),
82+
decision: (modelID: string | ModelID, providerOptions?: ProviderOptions) =>
83+
GoogleDecisions.route
84+
.with({ endpoint: { baseURL: media.baseURL }, auth: media.auth, headers: media.headers, http: media.http })
85+
.model<GoogleDecisions.OptionsInput>({ id: modelID, providerOptions }),
8086
configure,
8187
}
8288
}
@@ -92,6 +98,10 @@ export const embeddingModel = (modelID: string, settings?: Settings) => {
9298
const input = toConfig(settings)
9399
return configure(input).embedding(modelID, input.providerOptions)
94100
}
101+
export const decisionModel = (modelID: string, settings?: Settings) => {
102+
const input = toConfig(settings)
103+
return configure(input).decision(modelID, input.providerOptions)
104+
}
95105

96106
export const image = provider.image
97107
export const embedding = provider.embedding
@@ -100,3 +110,4 @@ export const interactions = provider.interactions
100110
export const video = provider.video
101111
export const speech = provider.speech
102112
export const transcription = provider.transcription
113+
export const decision = provider.decision
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
export { decisionModel as model } from "../google.js"
2+
export type { Settings } from "../google.js"

0 commit comments

Comments
 (0)