|
| 1 | +import { Effect, Schema } from "effect" |
| 2 | +import { DecisionResponse, DecisionUsage, type DecisionPart, type DecisionRequest } from "../decision.js" |
| 3 | +import type { Media } from "../media.js" |
| 4 | +import { DecisionRoute } from "../route/decision.js" |
| 5 | +import { Endpoint } from "../route/endpoint.js" |
| 6 | +import { MediaProtocol } from "../route/media-protocol.js" |
| 7 | +import { |
| 8 | + AIError, |
| 9 | + ProviderID, |
| 10 | + UnsupportedOperationError, |
| 11 | + mergeJsonRecords, |
| 12 | + type ProviderOptions, |
| 13 | +} from "../schema/index.js" |
| 14 | +import { encodeJson } from "../utils/json.js" |
| 15 | +import { JsonObject, knownString, ProviderShared } from "./shared.js" |
| 16 | +import { SystemOne } from "./system-one.js" |
| 17 | +import { MediaInput } from "./utils/media-input.js" |
| 18 | + |
| 19 | +const ID = "google-decisions" |
| 20 | +const NAME = "Google Decisions" |
| 21 | +export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta" |
| 22 | + |
| 23 | +const Options = Schema.Struct({ |
| 24 | + /** Token budget for every image and video input; `ultra_high` applies to images only. */ |
| 25 | + mediaResolution: Schema.optional(knownString<"low" | "medium" | "high" | "ultra_high">()), |
| 26 | +}) |
| 27 | +/** Known fields map to their wire names; anything else is sent as given. */ |
| 28 | +export type OptionsInput = ProviderOptions & typeof Options.Encoded |
| 29 | + |
| 30 | +const Usage = Schema.StructWithRest( |
| 31 | + Schema.Struct({ |
| 32 | + total_input_tokens: Schema.optional(Schema.Number), |
| 33 | + total_output_tokens: Schema.optional(Schema.Number), |
| 34 | + total_tokens: Schema.optional(Schema.Number), |
| 35 | + }), |
| 36 | + [JsonObject], |
| 37 | +) |
| 38 | +const Response = Schema.Struct({ |
| 39 | + answers: Schema.Record(Schema.String, SystemOne.Answer), |
| 40 | + usage: Schema.optional(Usage), |
| 41 | +}) |
| 42 | + |
| 43 | +const decodeOptions = ProviderShared.validateWith(Schema.decodeUnknownEffect(Options)) |
| 44 | + |
| 45 | +const fromRequest = Effect.fn("GoogleDecisions.fromRequest")(function* (request: DecisionRequest) { |
| 46 | + const options = yield* decodeOptions(request.providerOptions ?? {}) |
| 47 | + const { mediaResolution: _, ...native } = request.providerOptions ?? {} |
| 48 | + return { |
| 49 | + ...mergeJsonRecords(native, request.http?.body), |
| 50 | + model: request.model.id, |
| 51 | + input: separateText( |
| 52 | + yield* Effect.forEach(request.input, (part) => inputPart(request, part, options.mediaResolution)), |
| 53 | + ), |
| 54 | + questions: { |
| 55 | + ...(yield* SystemOne.encodeQuestions(request.questions)), |
| 56 | + // Google names System One's predicate criteria `true_description` and `false_description`. |
| 57 | + ...Object.fromEntries( |
| 58 | + request.questions.flatMap((question) => |
| 59 | + question.type === "predicate" && question.criteria !== undefined |
| 60 | + ? [ |
| 61 | + [ |
| 62 | + question.name, |
| 63 | + { |
| 64 | + type: "noul", |
| 65 | + instructions: question.instructions, |
| 66 | + criteria: { |
| 67 | + ...(question.criteria.true === undefined ? {} : { true_description: question.criteria.true }), |
| 68 | + ...(question.criteria.false === undefined ? {} : { false_description: question.criteria.false }), |
| 69 | + }, |
| 70 | + }, |
| 71 | + ], |
| 72 | + ] |
| 73 | + : [], |
| 74 | + ), |
| 75 | + ), |
| 76 | + }, |
| 77 | + } |
| 78 | +}) |
| 79 | + |
| 80 | +const inputPart = ( |
| 81 | + request: DecisionRequest, |
| 82 | + part: DecisionPart, |
| 83 | + resolution: string | undefined, |
| 84 | +): Effect.Effect<Record<string, unknown>, AIError> => { |
| 85 | + if (part.type === "text") return Effect.succeed({ type: "text", text: part.text }) |
| 86 | + // Google reads every text block as text; JSON is serialized so its keys are never parsed as content blocks. |
| 87 | + if (part.type === "json") return Effect.succeed({ type: "text", text: encodeJson(part.value) }) |
| 88 | + return mediaPart(request, part.media, resolution) |
| 89 | +} |
| 90 | + |
| 91 | +// Google joins adjacent text blocks with no separator, so each one after another starts on a new line. |
| 92 | +const separateText = (blocks: ReadonlyArray<Record<string, unknown>>) => |
| 93 | + blocks.map((block, index) => |
| 94 | + block.type === "text" && blocks[index - 1]?.type === "text" ? { ...block, text: `\n${block.text}` } : block, |
| 95 | + ) |
| 96 | + |
| 97 | +const mediaPart = (request: DecisionRequest, asset: Media.Asset, resolution: string | undefined) => { |
| 98 | + const type = |
| 99 | + asset.kind === "image" || asset.kind === "audio" || asset.kind === "video" |
| 100 | + ? asset.kind |
| 101 | + : asset.mediaType.toLowerCase() === "application/pdf" |
| 102 | + ? "document" |
| 103 | + : undefined |
| 104 | + if (type === undefined) |
| 105 | + return Effect.fail( |
| 106 | + new AIError({ |
| 107 | + reason: new UnsupportedOperationError({ |
| 108 | + operation: "decision.media", |
| 109 | + provider: request.model.provider, |
| 110 | + route: request.model.route.id, |
| 111 | + message: `${NAME} takes images, audio, video and PDFs; got ${asset.mediaType}`, |
| 112 | + }), |
| 113 | + }), |
| 114 | + ) |
| 115 | + const block = { |
| 116 | + type, |
| 117 | + mime_type: asset.mediaType.toLowerCase(), |
| 118 | + // `ultra_high` is image-only, so video keeps Google's default for it. |
| 119 | + ...(resolution !== undefined && (type === "image" || (type === "video" && resolution !== "ultra_high")) |
| 120 | + ? { resolution } |
| 121 | + : {}), |
| 122 | + } |
| 123 | + const inline = asset.inline() |
| 124 | + if (inline) return Effect.succeed({ ...block, data: inline.base64 }) |
| 125 | + // Google fetches URLs itself, so one that needs our download credentials can't be forwarded. |
| 126 | + if (asset.source.type === "url" && /^https:\/\//i.test(asset.source.url) && asset.headers === undefined) |
| 127 | + return Effect.succeed({ ...block, uri: asset.source.url }) |
| 128 | + const file = MediaInput.refID(asset, ProviderID.make("google")) |
| 129 | + if (file !== undefined) return Effect.succeed({ ...block, uri: file }) |
| 130 | + return Effect.fail( |
| 131 | + ProviderShared.invalidRequest( |
| 132 | + `${NAME} accepts inline media, public HTTPS URLs and Google Files API URIs; materialize ${asset.source.type} media first`, |
| 133 | + ), |
| 134 | + ) |
| 135 | +} |
| 136 | + |
| 137 | +const decodeDocument = MediaProtocol.identity({ id: ID, name: NAME, provider: "google" }).decodeJson(Response) |
| 138 | + |
| 139 | +export const protocol = DecisionRoute.protocol({ |
| 140 | + id: ID, |
| 141 | + name: NAME, |
| 142 | + body: { from: fromRequest }, |
| 143 | + response: { |
| 144 | + decode: Effect.fn("GoogleDecisions.decodeResponse")(function* (response, context) { |
| 145 | + const output = yield* decodeDocument(response) |
| 146 | + const data = output.value |
| 147 | + const answers = data.answers |
| 148 | + const names = context.request.questions.map((question) => question.name) |
| 149 | + if (Object.keys(answers).length !== names.length || !names.every((name) => Object.hasOwn(answers, name))) |
| 150 | + return yield* output.invalid(`${NAME} answers do not match the requested questions`) |
| 151 | + return new DecisionResponse({ |
| 152 | + answers: context.request.questions.map((question) => SystemOne.toAnswer(question, answers[question.name])), |
| 153 | + usage: |
| 154 | + data.usage === undefined |
| 155 | + ? undefined |
| 156 | + : new DecisionUsage({ |
| 157 | + inputTokens: data.usage.total_input_tokens, |
| 158 | + outputTokens: data.usage.total_output_tokens, |
| 159 | + totalTokens: ProviderShared.totalTokens( |
| 160 | + data.usage.total_input_tokens, |
| 161 | + data.usage.total_output_tokens, |
| 162 | + data.usage.total_tokens, |
| 163 | + ), |
| 164 | + }), |
| 165 | + providerMetadata: |
| 166 | + data.usage === undefined ? undefined : { [context.providerMetadataKey]: { usage: data.usage } }, |
| 167 | + }) |
| 168 | + }), |
| 169 | + }, |
| 170 | +}) |
| 171 | + |
| 172 | +export const route = DecisionRoute.make({ |
| 173 | + id: ID, |
| 174 | + provider: "google", |
| 175 | + protocol, |
| 176 | + endpoint: Endpoint.path("/decisions", { baseURL: DEFAULT_BASE_URL }), |
| 177 | +}) |
| 178 | + |
| 179 | +export * as GoogleDecisions from "./google-decisions.js" |
0 commit comments