diff --git a/api/v1alpha1/ai_service_backend.go b/api/v1alpha1/ai_service_backend.go index cddcc253f9..20c15a2c4a 100644 --- a/api/v1alpha1/ai_service_backend.go +++ b/api/v1alpha1/ai_service_backend.go @@ -76,10 +76,76 @@ type AIServiceBackendSpec struct { // +optional BodyMutation *HTTPBodyMutation `json:"bodyMutation,omitempty"` + // ModelTranslationHints declares translation hints for models routed through this backend, + // keyed by the exact ModelNameOverride used in an AIGatewayRouteRuleBackendRef that targets + // this backend. This is the single source of truth for a model's hints on this backend: any + // AIGatewayRoute, in any namespace, that references this backend with a matching + // ModelNameOverride picks up these hints without repeating them. + // + // Since an AIServiceBackend has exactly one Schema, hints declared here are inherently scoped + // to that provider — the same logical model hosted on a different AIServiceBackend (e.g. the + // same Claude model on AWS Bedrock vs. GCP Vertex AI) is configured as a separate entry on + // that other backend, which is correct because providers can genuinely support different + // features for what is nominally the same model. + // + // +optional + // +listType=map + // +listMapKey=modelNameOverride + ModelTranslationHints []ModelTranslationHintsEntry `json:"modelTranslationHints,omitempty"` + // TODO: maybe add backend-level LLMRequestCost configuration that overrides the AIGatewayRoute-level LLMRequestCost. // That may be useful for the backend that has a different cost calculation logic. } +// ModelTranslationHintsEntry associates ModelTranslationHints with a specific model routed +// through this backend. +type ModelTranslationHintsEntry struct { + // ModelNameOverride identifies which model these hints apply to. Must match the + // ModelNameOverride configured on an AIGatewayRouteRuleBackendRef that targets this backend + // for the hints to take effect. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + ModelNameOverride string `json:"modelNameOverride"` + + // ModelTranslationHints are the hints that apply to this model on this backend. + ModelTranslationHints `json:",inline"` +} + +// ModelTranslationHints provides model-specific hints to guide request and response translation. +// When set, translators use these values instead of model-name heuristics. All fields are +// optional; a nil field falls back to the built-in heuristic for backward compatibility. +// +// These hints are typically populated by a control plane from a model registry or catalog that +// tracks per-model metadata such as maximum output token limits and supported features. +type ModelTranslationHints struct { + // MaxOutputTokens is the maximum number of tokens the model can produce in a single response. + // When set, translators use this as the default max_tokens value for APIs that require it + // (e.g., the Anthropic Messages API) when the client request omits max_tokens. + // + // +optional + // +kubebuilder:validation:Minimum=1 + MaxOutputTokens *int64 `json:"maxOutputTokens,omitempty"` + + // SupportsResponseJSONSchema indicates whether the model accepts a JSON Schema object in + // response_format (as opposed to only {"type": "json_object"}). + // + // +optional + SupportsResponseJSONSchema *bool `json:"supportsResponseJsonSchema,omitempty"` + + // SupportsReasoningEffort indicates whether the model accepts a reasoning effort parameter + // (Gemini thinking level / Anthropic output_config.effort). + // + // +optional + SupportsReasoningEffort *bool `json:"supportsReasoningEffort,omitempty"` + + // SupportsOutputConfig indicates whether the model accepts an Anthropic-style output_config + // field for structured output (e.g., Claude Opus/Sonnet 4.5 and later). + // + // +optional + SupportsOutputConfig *bool `json:"supportsOutputConfig,omitempty"` +} + // HTTPHeaderMutation defines the mutation of HTTP headers that will be applied to the request type HTTPHeaderMutation struct { // Set overwrites/adds the request with the given header (name, value) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index f795d34163..c6c91a9733 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -338,6 +338,13 @@ func (in *AIServiceBackendSpec) DeepCopyInto(out *AIServiceBackendSpec) { *out = new(HTTPBodyMutation) (*in).DeepCopyInto(*out) } + if in.ModelTranslationHints != nil { + in, out := &in.ModelTranslationHints, &out.ModelTranslationHints + *out = make([]ModelTranslationHintsEntry, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AIServiceBackendSpec. @@ -1615,6 +1622,57 @@ func (in *MCPToolFilter) DeepCopy() *MCPToolFilter { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelTranslationHints) DeepCopyInto(out *ModelTranslationHints) { + *out = *in + if in.MaxOutputTokens != nil { + in, out := &in.MaxOutputTokens, &out.MaxOutputTokens + *out = new(int64) + **out = **in + } + if in.SupportsResponseJSONSchema != nil { + in, out := &in.SupportsResponseJSONSchema, &out.SupportsResponseJSONSchema + *out = new(bool) + **out = **in + } + if in.SupportsReasoningEffort != nil { + in, out := &in.SupportsReasoningEffort, &out.SupportsReasoningEffort + *out = new(bool) + **out = **in + } + if in.SupportsOutputConfig != nil { + in, out := &in.SupportsOutputConfig, &out.SupportsOutputConfig + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelTranslationHints. +func (in *ModelTranslationHints) DeepCopy() *ModelTranslationHints { + if in == nil { + return nil + } + out := new(ModelTranslationHints) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelTranslationHintsEntry) DeepCopyInto(out *ModelTranslationHintsEntry) { + *out = *in + in.ModelTranslationHints.DeepCopyInto(&out.ModelTranslationHints) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelTranslationHintsEntry. +func (in *ModelTranslationHintsEntry) DeepCopy() *ModelTranslationHintsEntry { + if in == nil { + return nil + } + out := new(ModelTranslationHintsEntry) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PerModelQuota) DeepCopyInto(out *PerModelQuota) { *out = *in diff --git a/api/v1beta1/ai_service_backend.go b/api/v1beta1/ai_service_backend.go index ed0d3b8573..80c1be1d4a 100644 --- a/api/v1beta1/ai_service_backend.go +++ b/api/v1beta1/ai_service_backend.go @@ -91,6 +91,72 @@ type AIServiceBackendSpec struct { // +kubebuilder:validation:MaxItems=16 HeaderValueFilters []HTTPHeaderValueFilter `json:"headerValueFilters,omitempty"` + // ModelTranslationHints declares translation hints for models routed through this backend, + // keyed by the exact ModelNameOverride used in an AIGatewayRouteRuleBackendRef that targets + // this backend. This is the single source of truth for a model's hints on this backend: any + // AIGatewayRoute, in any namespace, that references this backend with a matching + // ModelNameOverride picks up these hints without repeating them. + // + // Since an AIServiceBackend has exactly one Schema, hints declared here are inherently scoped + // to that provider — the same logical model hosted on a different AIServiceBackend (e.g. the + // same Claude model on AWS Bedrock vs. GCP Vertex AI) is configured as a separate entry on + // that other backend, which is correct because providers can genuinely support different + // features for what is nominally the same model. + // + // +optional + // +listType=map + // +listMapKey=modelNameOverride + ModelTranslationHints []ModelTranslationHintsEntry `json:"modelTranslationHints,omitempty"` + // TODO: maybe add backend-level LLMRequestCost configuration that overrides the AIGatewayRoute-level LLMRequestCost. // That may be useful for the backend that has a different cost calculation logic. } + +// ModelTranslationHintsEntry associates ModelTranslationHints with a specific model routed +// through this backend. +type ModelTranslationHintsEntry struct { + // ModelNameOverride identifies which model these hints apply to. Must match the + // ModelNameOverride configured on an AIGatewayRouteRuleBackendRef that targets this backend + // for the hints to take effect. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + ModelNameOverride string `json:"modelNameOverride"` + + // ModelTranslationHints are the hints that apply to this model on this backend. + ModelTranslationHints `json:",inline"` +} + +// ModelTranslationHints provides model-specific hints to guide request and response translation. +// When set, translators use these values instead of model-name heuristics. All fields are +// optional; a nil field falls back to the built-in heuristic for backward compatibility. +// +// These hints are typically populated by a control plane from a model registry or catalog that +// tracks per-model metadata such as maximum output token limits and supported features. +type ModelTranslationHints struct { + // MaxOutputTokens is the maximum number of tokens the model can produce in a single response. + // When set, translators use this as the default max_tokens value for APIs that require it + // (e.g., the Anthropic Messages API) when the client request omits max_tokens. + // + // +optional + // +kubebuilder:validation:Minimum=1 + MaxOutputTokens *int64 `json:"maxOutputTokens,omitempty"` + + // SupportsResponseJSONSchema indicates whether the model accepts a JSON Schema object in + // response_format (as opposed to only {"type": "json_object"}). + // + // +optional + SupportsResponseJSONSchema *bool `json:"supportsResponseJsonSchema,omitempty"` + + // SupportsReasoningEffort indicates whether the model accepts a reasoning effort parameter + // (Gemini thinking level / Anthropic output_config.effort). + // + // +optional + SupportsReasoningEffort *bool `json:"supportsReasoningEffort,omitempty"` + + // SupportsOutputConfig indicates whether the model accepts an Anthropic-style output_config + // field for structured output (e.g., Claude Opus/Sonnet 4.5 and later). + // + // +optional + SupportsOutputConfig *bool `json:"supportsOutputConfig,omitempty"` +} diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 8a8c91f456..5f2252d4ba 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -345,6 +345,13 @@ func (in *AIServiceBackendSpec) DeepCopyInto(out *AIServiceBackendSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.ModelTranslationHints != nil { + in, out := &in.ModelTranslationHints, &out.ModelTranslationHints + *out = make([]ModelTranslationHintsEntry, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AIServiceBackendSpec. @@ -1772,6 +1779,57 @@ func (in *MCPToolFilter) DeepCopy() *MCPToolFilter { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelTranslationHints) DeepCopyInto(out *ModelTranslationHints) { + *out = *in + if in.MaxOutputTokens != nil { + in, out := &in.MaxOutputTokens, &out.MaxOutputTokens + *out = new(int64) + **out = **in + } + if in.SupportsResponseJSONSchema != nil { + in, out := &in.SupportsResponseJSONSchema, &out.SupportsResponseJSONSchema + *out = new(bool) + **out = **in + } + if in.SupportsReasoningEffort != nil { + in, out := &in.SupportsReasoningEffort, &out.SupportsReasoningEffort + *out = new(bool) + **out = **in + } + if in.SupportsOutputConfig != nil { + in, out := &in.SupportsOutputConfig, &out.SupportsOutputConfig + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelTranslationHints. +func (in *ModelTranslationHints) DeepCopy() *ModelTranslationHints { + if in == nil { + return nil + } + out := new(ModelTranslationHints) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelTranslationHintsEntry) DeepCopyInto(out *ModelTranslationHintsEntry) { + *out = *in + in.ModelTranslationHints.DeepCopyInto(&out.ModelTranslationHints) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelTranslationHintsEntry. +func (in *ModelTranslationHintsEntry) DeepCopy() *ModelTranslationHintsEntry { + if in == nil { + return nil + } + out := new(ModelTranslationHintsEntry) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProtectedResourceMetadata) DeepCopyInto(out *ProtectedResourceMetadata) { *out = *in diff --git a/internal/controller/gateway.go b/internal/controller/gateway.go index 5cb2630cf6..c995fe65bf 100644 --- a/internal/controller/gateway.go +++ b/internal/controller/gateway.go @@ -239,6 +239,29 @@ func bodyMutationToFilterAPI(m *aigv1b1.HTTPBodyMutation) *filterapi.HTTPBodyMut return ret } +// modelTranslationHintsForModel finds the ModelTranslationHintsEntry in entries whose +// ModelNameOverride exactly matches modelNameOverride and converts it to filterapi.ModelTranslationHints. +// Returns nil if modelNameOverride is empty or no entry matches, in which case translators fall +// back to their built-in heuristics. +func modelTranslationHintsForModel(entries []aigv1b1.ModelTranslationHintsEntry, modelNameOverride string) *filterapi.ModelTranslationHints { + if modelNameOverride == "" { + return nil + } + for _, entry := range entries { + if entry.ModelNameOverride != modelNameOverride { + continue + } + h := entry.ModelTranslationHints + return &filterapi.ModelTranslationHints{ + MaxOutputTokens: h.MaxOutputTokens, + SupportsResponseJSONSchema: h.SupportsResponseJSONSchema, + SupportsReasoningEffort: h.SupportsReasoningEffort, + SupportsOutputConfig: h.SupportsOutputConfig, + } + } + return nil +} + // validateCELExpression validates and returns a CEL expression for cost calculation. func validateCELExpression(cost aigv1b1.LLMRequestCost) (string, error) { if cost.CEL == nil { @@ -551,6 +574,8 @@ func (c *GatewayController) reconcileFilterConfigSecret( mergedBodyMutation := mergeBodyMutations(routeBodyMutation, backendBodyMutation) b.BodyMutation = bodyMutationToFilterAPI(mergedBodyMutation) + b.TranslationHints = modelTranslationHintsForModel(backendObj.Spec.ModelTranslationHints, backendRef.ModelNameOverride) + b.Schema = schemaToFilterAPI(backendObj.Spec.APISchema) b.HeaderValueFilters = headerValueFiltersToFilterAPI(backendObj.Spec.HeaderValueFilters) } diff --git a/internal/controller/gateway_test.go b/internal/controller/gateway_test.go index e69e6485c8..52b00710fb 100644 --- a/internal/controller/gateway_test.go +++ b/internal/controller/gateway_test.go @@ -3953,6 +3953,81 @@ func Test_bodyMutationToFilterAPI(t *testing.T) { } } +func Test_modelTranslationHintsForModel(t *testing.T) { + opusHints := aigv1b1.ModelTranslationHints{ + MaxOutputTokens: ptr.To(int64(32000)), + SupportsResponseJSONSchema: ptr.To(true), + SupportsReasoningEffort: ptr.To(true), + SupportsOutputConfig: ptr.To(false), + } + geminiHints := aigv1b1.ModelTranslationHints{ + MaxOutputTokens: ptr.To(int64(65536)), + } + entries := []aigv1b1.ModelTranslationHintsEntry{ + {ModelNameOverride: "claude-opus-4-6@20250514", ModelTranslationHints: opusHints}, + {ModelNameOverride: "gemini-3-pro", ModelTranslationHints: geminiHints}, + } + + tests := []struct { + name string + entries []aigv1b1.ModelTranslationHintsEntry + modelNameOverride string + expected *filterapi.ModelTranslationHints + }{ + { + name: "empty modelNameOverride", + entries: entries, + modelNameOverride: "", + expected: nil, + }, + { + name: "no entries", + entries: nil, + modelNameOverride: "claude-opus-4-6@20250514", + expected: nil, + }, + { + name: "no matching entry", + entries: entries, + modelNameOverride: "claude-sonnet-4-6@20250514", + expected: nil, + }, + { + name: "matches the first entry among multiple", + entries: entries, + modelNameOverride: "claude-opus-4-6@20250514", + expected: &filterapi.ModelTranslationHints{ + MaxOutputTokens: ptr.To(int64(32000)), + SupportsResponseJSONSchema: ptr.To(true), + SupportsReasoningEffort: ptr.To(true), + SupportsOutputConfig: ptr.To(false), + }, + }, + { + name: "matches a later entry among multiple", + entries: entries, + modelNameOverride: "gemini-3-pro", + expected: &filterapi.ModelTranslationHints{ + MaxOutputTokens: ptr.To(int64(65536)), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := modelTranslationHintsForModel(tt.entries, tt.modelNameOverride) + if tt.expected == nil { + require.Nil(t, result) + return + } + require.NotNil(t, result) + if d := cmp.Diff(tt.expected, result); d != "" { + t.Errorf("modelTranslationHintsForModel() mismatch (-expected +got):\n%s", d) + } + }) + } +} + // TestGatewayController_reconcileFilterConfigSecret_GlobalDefaults tests that // global LLM request costs from GatewayConfig are properly included in the filter config // when no routes override them. diff --git a/internal/endpointspec/endpointspec.go b/internal/endpointspec/endpointspec.go index 37e8cbe868..2ffdf5a716 100644 --- a/internal/endpointspec/endpointspec.go +++ b/internal/endpointspec/endpointspec.go @@ -61,17 +61,19 @@ type ( // * mutatedBody: The possibly mutated request body as a byte slice. Or nil if no mutation is needed. // * err: An error if parsing fails. ParseBody(body []byte, costConfigured bool) (originalModel internalapi.OriginalModel, req *ReqT, stream bool, mutatedBody []byte, err error) - // GetTranslator selects the appropriate translator based on the output API schema - // and an optional model name override. + // GetTranslator selects the appropriate translator for the given backend. + // + // The full backend is passed (rather than just the schema and model name override) so that + // translators can access model-specific translation hints alongside the schema and model name. // // Parameters: - // * out: The output API schema for which the translator is needed. - // * modelNameOverride: An optional model name to override the one specified in the request. + // * backend: The backend for which the translator is needed. Its Schema selects the translator, + // while ModelNameOverride and TranslationHints tune its behavior. // // Returns: // * translator: The selected translator of type Translator[ReqT, RespT, RespChunkT]. // * err: An error if translator selection fails. - GetTranslator(schema filterapi.VersionedAPISchema, modelNameOverride string) (translator.Translator[ReqT, tracingapi.Span[RespT, RespChunkT]], error) + GetTranslator(backend *filterapi.Backend) (translator.Translator[ReqT, tracingapi.Span[RespT, RespChunkT]], error) // RedactSensitiveInfoFromRequest creates a redacted copy of the request for safe debug logging. // Sensitive content (messages, images, audio, tool parameters, etc.) is replaced with placeholders // containing length and hash information to aid in debugging cache hits/misses and correlation. @@ -199,7 +201,9 @@ func (ChatCompletionsEndpointSpec) ParseMultipartBody([]byte, string, bool) (int } // GetTranslator implements [EndpointSpec.GetTranslator]. -func (ChatCompletionsEndpointSpec) GetTranslator(schema filterapi.VersionedAPISchema, modelNameOverride string) (translator.OpenAIChatCompletionTranslator, error) { +func (ChatCompletionsEndpointSpec) GetTranslator(backend *filterapi.Backend) (translator.OpenAIChatCompletionTranslator, error) { + schema := backend.Schema + modelNameOverride := backend.ModelNameOverride switch schema.Name { case filterapi.APISchemaOpenAI: return translator.NewChatCompletionOpenAIToOpenAITranslator(schema.OpenAIPrefix(), modelNameOverride), nil @@ -208,13 +212,13 @@ func (ChatCompletionsEndpointSpec) GetTranslator(schema filterapi.VersionedAPISc case filterapi.APISchemaAWSBedrock: return translator.NewChatCompletionOpenAIToAWSBedrockTranslator(modelNameOverride), nil case filterapi.APISchemaAWSAnthropic: - return translator.NewChatCompletionOpenAIToAWSAnthropicTranslator(schema.Version, modelNameOverride), nil + return translator.NewChatCompletionOpenAIToAWSAnthropicTranslator(schema.Version, modelNameOverride, backend.TranslationHints), nil case filterapi.APISchemaAzureOpenAI: return translator.NewChatCompletionOpenAIToAzureOpenAITranslator(schema.Version, modelNameOverride), nil case filterapi.APISchemaGCPVertexAI: - return translator.NewChatCompletionOpenAIToGCPVertexAITranslator(modelNameOverride), nil + return translator.NewChatCompletionOpenAIToGCPVertexAITranslator(modelNameOverride, backend.TranslationHints), nil case filterapi.APISchemaGCPAnthropic: - return translator.NewChatCompletionOpenAIToGCPAnthropicTranslator(schema.Version, modelNameOverride), nil + return translator.NewChatCompletionOpenAIToGCPAnthropicTranslator(schema.Version, modelNameOverride, backend.TranslationHints), nil default: return nil, fmt.Errorf("%w: unsupported API schema: backend=%s", internalapi.ErrInvalidRequestBody, schema) } @@ -263,10 +267,11 @@ func (CompletionsEndpointSpec) ParseMultipartBody([]byte, string, bool) (interna } // GetTranslator implements [EndpointSpec.GetTranslator]. -func (CompletionsEndpointSpec) GetTranslator(schema filterapi.VersionedAPISchema, modelNameOverride string) (translator.OpenAICompletionTranslator, error) { +func (CompletionsEndpointSpec) GetTranslator(backend *filterapi.Backend) (translator.OpenAICompletionTranslator, error) { + schema := backend.Schema switch schema.Name { case filterapi.APISchemaOpenAI: - return translator.NewCompletionOpenAIToOpenAITranslator(schema.OpenAIPrefix(), modelNameOverride), nil + return translator.NewCompletionOpenAIToOpenAITranslator(schema.OpenAIPrefix(), backend.ModelNameOverride), nil default: return nil, fmt.Errorf("%w: unsupported API schema: backend=%s", internalapi.ErrInvalidRequestBody, schema) } @@ -300,7 +305,9 @@ func (EmbeddingsEndpointSpec) ParseMultipartBody([]byte, string, bool) (internal } // GetTranslator implements [EndpointSpec.GetTranslator]. -func (EmbeddingsEndpointSpec) GetTranslator(schema filterapi.VersionedAPISchema, modelNameOverride string) (translator.OpenAIEmbeddingTranslator, error) { +func (EmbeddingsEndpointSpec) GetTranslator(backend *filterapi.Backend) (translator.OpenAIEmbeddingTranslator, error) { + schema := backend.Schema + modelNameOverride := backend.ModelNameOverride switch schema.Name { case filterapi.APISchemaOpenAI: return translator.NewEmbeddingOpenAIToOpenAITranslator(schema.OpenAIPrefix(), modelNameOverride), nil @@ -355,10 +362,11 @@ func (ImageGenerationEndpointSpec) ParseMultipartBody([]byte, string, bool) (int } // GetTranslator implements [EndpointSpec.GetTranslator]. -func (ImageGenerationEndpointSpec) GetTranslator(schema filterapi.VersionedAPISchema, modelNameOverride string) (translator.OpenAIImageGenerationTranslator, error) { +func (ImageGenerationEndpointSpec) GetTranslator(backend *filterapi.Backend) (translator.OpenAIImageGenerationTranslator, error) { + schema := backend.Schema switch schema.Name { case filterapi.APISchemaOpenAI: - return translator.NewImageGenerationOpenAIToOpenAITranslator(schema.OpenAIPrefix(), modelNameOverride), nil + return translator.NewImageGenerationOpenAIToOpenAITranslator(schema.OpenAIPrefix(), backend.ModelNameOverride), nil default: return nil, fmt.Errorf("%w: unsupported API schema: backend=%s", internalapi.ErrInvalidRequestBody, schema) } @@ -390,14 +398,15 @@ func (ResponsesEndpointSpec) ParseMultipartBody([]byte, string, bool) (internala } // GetTranslator implements [EndpointSpec.GetTranslator]. -func (ResponsesEndpointSpec) GetTranslator(schema filterapi.VersionedAPISchema, modelNameOverride string) (translator.OpenAIResponsesTranslator, error) { +func (ResponsesEndpointSpec) GetTranslator(backend *filterapi.Backend) (translator.OpenAIResponsesTranslator, error) { + schema := backend.Schema switch schema.Name { case filterapi.APISchemaOpenAI: - return translator.NewResponsesOpenAIToOpenAITranslator(schema.OpenAIPrefix(), modelNameOverride), nil + return translator.NewResponsesOpenAIToOpenAITranslator(schema.OpenAIPrefix(), backend.ModelNameOverride), nil case filterapi.APISchemaAWSOpenAI: - return translator.NewResponsesOpenAIToAWSOpenAITranslator(schema.OpenAIPrefix(), modelNameOverride), nil + return translator.NewResponsesOpenAIToAWSOpenAITranslator(schema.OpenAIPrefix(), backend.ModelNameOverride), nil case filterapi.APISchemaAzureOpenAI: - return translator.NewResponsesOpenAIToAzureOpenAITranslator(schema.Version, modelNameOverride), nil + return translator.NewResponsesOpenAIToAzureOpenAITranslator(schema.Version, backend.ModelNameOverride), nil default: return nil, fmt.Errorf("%w: unsupported API schema: backend=%s", internalapi.ErrInvalidRequestBody, schema) } @@ -441,7 +450,9 @@ func (MessagesEndpointSpec) ParseMultipartBody([]byte, string, bool) (internalap } // GetTranslator implements [EndpointSpec.GetTranslator]. -func (MessagesEndpointSpec) GetTranslator(schema filterapi.VersionedAPISchema, modelNameOverride string) (translator.AnthropicMessagesTranslator, error) { +func (MessagesEndpointSpec) GetTranslator(backend *filterapi.Backend) (translator.AnthropicMessagesTranslator, error) { + schema := backend.Schema + modelNameOverride := backend.ModelNameOverride // Messages processor only supports Anthropic-native translators. switch schema.Name { case filterapi.APISchemaGCPAnthropic: @@ -495,7 +506,9 @@ func (MessagesCountTokensEndpointSpec) ParseBody( } // GetTranslator implements [EndpointSpec.GetTranslator]. -func (MessagesCountTokensEndpointSpec) GetTranslator(schema filterapi.VersionedAPISchema, modelNameOverride string) (translator.AnthropicCountTokensTranslator, error) { +func (MessagesCountTokensEndpointSpec) GetTranslator(backend *filterapi.Backend) (translator.AnthropicCountTokensTranslator, error) { + schema := backend.Schema + modelNameOverride := backend.ModelNameOverride switch schema.Name { case filterapi.APISchemaGCPAnthropic: return translator.NewCountTokensToGCPAnthropicTranslator(schema.Version, modelNameOverride), nil @@ -536,10 +549,11 @@ func (RerankEndpointSpec) ParseMultipartBody([]byte, string, bool) (internalapi. } // GetTranslator implements [EndpointSpec.GetTranslator]. -func (RerankEndpointSpec) GetTranslator(schema filterapi.VersionedAPISchema, modelNameOverride string) (translator.CohereRerankTranslator, error) { +func (RerankEndpointSpec) GetTranslator(backend *filterapi.Backend) (translator.CohereRerankTranslator, error) { + schema := backend.Schema switch schema.Name { case filterapi.APISchemaCohere: - return translator.NewRerankCohereToCohereTranslator(schema.Version, modelNameOverride), nil + return translator.NewRerankCohereToCohereTranslator(schema.Version, backend.ModelNameOverride), nil default: return nil, fmt.Errorf("%w: unsupported API schema: backend=%s", internalapi.ErrInvalidRequestBody, schema) } @@ -575,10 +589,11 @@ func (SystemOneEndpointSpec) ParseMultipartBody([]byte, string, bool) (internala } // GetTranslator implements [EndpointSpec.GetTranslator]. -func (SystemOneEndpointSpec) GetTranslator(schema filterapi.VersionedAPISchema, modelNameOverride string) (translator.TypeSafeSystemOneTranslator, error) { +func (SystemOneEndpointSpec) GetTranslator(backend *filterapi.Backend) (translator.TypeSafeSystemOneTranslator, error) { + schema := backend.Schema switch schema.Name { case filterapi.APISchemaTypeSafe: - return translator.NewSystemOneTypeSafeToTypeSafeTranslator(schema.Version, modelNameOverride), nil + return translator.NewSystemOneTypeSafeToTypeSafeTranslator(schema.Version, backend.ModelNameOverride), nil default: return nil, fmt.Errorf("%w: unsupported API schema: backend=%s", internalapi.ErrInvalidRequestBody, schema) } @@ -641,7 +656,9 @@ func (TokenizeEndpointSpec) ParseBody( } // GetTranslator implements [EndpointSpec.GetTranslator]. -func (TokenizeEndpointSpec) GetTranslator(schema filterapi.VersionedAPISchema, modelNameOverride string) (translator.TokenizeTranslator, error) { +func (TokenizeEndpointSpec) GetTranslator(backend *filterapi.Backend) (translator.TokenizeTranslator, error) { + schema := backend.Schema + modelNameOverride := backend.ModelNameOverride switch schema.Name { case filterapi.APISchemaOpenAI: return translator.NewTokenizeTranslator(modelNameOverride), nil @@ -935,14 +952,14 @@ func (SpeechEndpointSpec) ParseMultipartBody([]byte, string, bool) (internalapi. // GetTranslator implements [EndpointSpec.GetTranslator]. func (SpeechEndpointSpec) GetTranslator( - schema filterapi.VersionedAPISchema, - modelNameOverride string, + backend *filterapi.Backend, ) (translator.OpenAISpeechTranslator, error) { + schema := backend.Schema switch schema.Name { case filterapi.APISchemaOpenAI: return translator.NewSpeechOpenAIToOpenAITranslator( schema.OpenAIPrefix(), - modelNameOverride, + backend.ModelNameOverride, ), nil default: return nil, fmt.Errorf("%w: unsupported API schema for speech: backend=%s", internalapi.ErrInvalidRequestBody, schema) @@ -1070,11 +1087,12 @@ func (TranscriptionEndpointSpec) ParseMultipartBody( // GetTranslator implements [Spec.GetTranslator]. func (TranscriptionEndpointSpec) GetTranslator( - schema filterapi.VersionedAPISchema, modelNameOverride string, + backend *filterapi.Backend, ) (translator.OpenAIAudioTranscriptionTranslator, error) { + schema := backend.Schema switch schema.Name { case filterapi.APISchemaOpenAI: - return translator.NewTranscriptionOpenAIToOpenAITranslator(schema.OpenAIPrefix(), modelNameOverride), nil + return translator.NewTranscriptionOpenAIToOpenAITranslator(schema.OpenAIPrefix(), backend.ModelNameOverride), nil default: return nil, fmt.Errorf("%w: unsupported API schema for audio transcription: backend=%s", internalapi.ErrInvalidRequestBody, schema) } @@ -1175,11 +1193,12 @@ func (TranslationEndpointSpec) ParseMultipartBody( // GetTranslator implements [Spec.GetTranslator]. func (TranslationEndpointSpec) GetTranslator( - schema filterapi.VersionedAPISchema, modelNameOverride string, + backend *filterapi.Backend, ) (translator.OpenAIAudioTranslationTranslator, error) { + schema := backend.Schema switch schema.Name { case filterapi.APISchemaOpenAI: - return translator.NewTranslationOpenAIToOpenAITranslator(schema.OpenAIPrefix(), modelNameOverride), nil + return translator.NewTranslationOpenAIToOpenAITranslator(schema.OpenAIPrefix(), backend.ModelNameOverride), nil default: return nil, fmt.Errorf("%w: unsupported API schema for audio translation: backend=%s", internalapi.ErrInvalidRequestBody, schema) } @@ -1207,13 +1226,14 @@ func (ResponsesInputTokensEndpointSpec) ParseBody( // GetTranslator implements [EndpointSpec.GetTranslator]. func (ResponsesInputTokensEndpointSpec) GetTranslator( - schema filterapi.VersionedAPISchema, modelNameOverride string, + backend *filterapi.Backend, ) (translator.OpenAIResponsesInputTokensTranslator, error) { + schema := backend.Schema switch schema.Name { case filterapi.APISchemaOpenAI: - return translator.NewResponsesInputTokensOpenAIToOpenAITranslator(schema.OpenAIPrefix(), modelNameOverride), nil + return translator.NewResponsesInputTokensOpenAIToOpenAITranslator(schema.OpenAIPrefix(), backend.ModelNameOverride), nil case filterapi.APISchemaAzureOpenAI: - return translator.NewResponsesInputTokensOpenAIToAzureOpenAITranslator(schema.Version, modelNameOverride), nil + return translator.NewResponsesInputTokensOpenAIToAzureOpenAITranslator(schema.Version, backend.ModelNameOverride), nil default: return nil, fmt.Errorf("%w: unsupported API schema for /v1/responses/input_tokens: backend=%s", internalapi.ErrInvalidRequestBody, schema) } diff --git a/internal/endpointspec/endpointspec_test.go b/internal/endpointspec/endpointspec_test.go index b5644c1a1d..5ca070cd1e 100644 --- a/internal/endpointspec/endpointspec_test.go +++ b/internal/endpointspec/endpointspec_test.go @@ -148,19 +148,19 @@ func TestChatCompletionsEndpointSpec_GetTranslator(t *testing.T) { s := schema t.Run("supported_"+string(s.Name), func(t *testing.T) { t.Parallel() - translator, err := spec.GetTranslator(s, "override") + translator, err := spec.GetTranslator(&filterapi.Backend{Schema: s, ModelNameOverride: "override"}) require.NoError(t, err) require.NotNil(t, translator) }) } t.Run("unsupported", func(t *testing.T) { - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: "Unknown"}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: "Unknown"}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "unsupported API schema") }) t.Run("AWSOpenAI request", func(t *testing.T) { - awsTranslator, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaAWSOpenAI}, "") + awsTranslator, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAWSOpenAI}}) require.NoError(t, err) original := []byte(`{"model":"us.openai.gpt-5.6-luna","messages":[]}`) headers, body, err := awsTranslator.RequestBody(original, &openai.ChatCompletionRequest{Model: "us.openai.gpt-5.6-luna"}, false) @@ -195,10 +195,10 @@ func TestCompletionsEndpointSpec_ParseBody(t *testing.T) { func TestCompletionsEndpointSpec_GetTranslator(t *testing.T) { spec := CompletionsEndpointSpec{} - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, ModelNameOverride: "override"}) require.NoError(t, err) - _, err = spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaAWSBedrock}, "override") + _, err = spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAWSBedrock}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "unsupported API schema") } @@ -254,14 +254,14 @@ func TestEmbeddingsEndpointSpec_GetTranslator(t *testing.T) { s := schema t.Run("supported_"+string(s.Name), func(t *testing.T) { t.Parallel() - tr, err := spec.GetTranslator(s, "override") + tr, err := spec.GetTranslator(&filterapi.Backend{Schema: s, ModelNameOverride: "override"}) require.NoError(t, err) require.NotNil(t, tr) }) } t.Run("unsupported", func(t *testing.T) { - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaCohere}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaCohere}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "unsupported API schema") }) } @@ -290,10 +290,10 @@ func TestImageGenerationEndpointSpec_ParseBody(t *testing.T) { func TestImageGenerationEndpointSpec_GetTranslator(t *testing.T) { spec := ImageGenerationEndpointSpec{} - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, ModelNameOverride: "override"}) require.NoError(t, err) - _, err = spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaAzureOpenAI}, "override") + _, err = spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAzureOpenAI}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "unsupported API schema") } @@ -335,12 +335,12 @@ func TestMessagesEndpointSpec_GetTranslator(t *testing.T) { {Name: filterapi.APISchemaOpenAI}, // This is for OpenAI-schema backends like vLLM that support the /v1/messages endpoint {Name: filterapi.APISchemaAWSBedrock}, // This is for AWS Bedrock translation from the /v1/messages endpoint } { - translator, err := spec.GetTranslator(schema, "override") + translator, err := spec.GetTranslator(&filterapi.Backend{Schema: schema, ModelNameOverride: "override"}) require.NoError(t, err) require.NotNil(t, translator) } - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaCohere}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaCohere}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "only supports") } @@ -368,10 +368,10 @@ func TestRerankEndpointSpec_ParseBody(t *testing.T) { func TestRerankEndpointSpec_GetTranslator(t *testing.T) { spec := RerankEndpointSpec{} - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaCohere}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaCohere}, ModelNameOverride: "override"}) require.NoError(t, err) - _, err = spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, "override") + _, err = spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "unsupported API schema") } @@ -403,10 +403,10 @@ func TestSystemOneEndpointSpec_ParseBody(t *testing.T) { func TestSystemOneEndpointSpec_GetTranslator(t *testing.T) { spec := SystemOneEndpointSpec{} - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaTypeSafe, Version: "v1"}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaTypeSafe, Version: "v1"}, ModelNameOverride: "override"}) require.NoError(t, err) - _, err = spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, "override") + _, err = spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "unsupported API schema") } @@ -477,13 +477,13 @@ func TestResponsesEndpointSpec_ParseBody(t *testing.T) { func TestResponsesEndpointSpec_GetTranslator(t *testing.T) { spec := ResponsesEndpointSpec{} - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, ModelNameOverride: "override"}) require.NoError(t, err) - _, err = spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaAzureOpenAI}, "override") + _, err = spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAzureOpenAI}, ModelNameOverride: "override"}) require.NoError(t, err) - awsTranslator, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaAWSOpenAI}, "us.openai.gpt-5.6-luna") + awsTranslator, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAWSOpenAI}, ModelNameOverride: "us.openai.gpt-5.6-luna"}) require.NoError(t, err) headers, body, err := awsTranslator.RequestBody( []byte(`{"model":"gpt-5.6-luna","input":"hello"}`), @@ -494,7 +494,7 @@ func TestResponsesEndpointSpec_GetTranslator(t *testing.T) { require.Equal(t, internalapi.Header{":path", "/openai/v1/responses"}, headers[0]) require.JSONEq(t, `{"model":"us.openai.gpt-5.6-luna","input":"hello"}`, string(body)) - awsTranslator, err = spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaAWSOpenAI}, "") + awsTranslator, err = spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAWSOpenAI}}) require.NoError(t, err) original := []byte(`{"model":"us.openai.gpt-5.6-luna","input":"hello"}`) _, body, err = awsTranslator.RequestBody(original, &openai.ResponseRequest{Model: "us.openai.gpt-5.6-luna"}, false) @@ -608,14 +608,14 @@ func TestTokenizeEndpointSpec_GetTranslator(t *testing.T) { s := schema t.Run("supported_"+string(s.Name), func(t *testing.T) { t.Parallel() - translator, err := spec.GetTranslator(s, "override") + translator, err := spec.GetTranslator(&filterapi.Backend{Schema: s, ModelNameOverride: "override"}) require.NoError(t, err) require.NotNil(t, translator) }) } t.Run("unsupported", func(t *testing.T) { - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: "Unknown"}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: "Unknown"}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "unsupported API schema for tokenize endpoint") }) } @@ -1190,10 +1190,10 @@ func TestSpeechEndpointSpec_ParseBody(t *testing.T) { func TestSpeechEndpointSpec_GetTranslator(t *testing.T) { spec := SpeechEndpointSpec{} - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, ModelNameOverride: "override"}) require.NoError(t, err) - _, err = spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaAzureOpenAI}, "override") + _, err = spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAzureOpenAI}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "unsupported API schema for speech") } @@ -1450,10 +1450,10 @@ func TestTranscriptionEndpointSpec_ParseMultipartBody(t *testing.T) { func TestTranscriptionEndpointSpec_GetTranslator(t *testing.T) { spec := TranscriptionEndpointSpec{} - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, ModelNameOverride: "override"}) require.NoError(t, err) - _, err = spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaAzureOpenAI}, "override") + _, err = spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAzureOpenAI}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "unsupported API schema for audio transcription") } @@ -1558,10 +1558,10 @@ func TestTranslationEndpointSpec_ParseMultipartBody(t *testing.T) { func TestTranslationEndpointSpec_GetTranslator(t *testing.T) { spec := TranslationEndpointSpec{} - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, ModelNameOverride: "override"}) require.NoError(t, err) - _, err = spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaAzureOpenAI}, "override") + _, err = spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAzureOpenAI}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "unsupported API schema for audio translation") } @@ -1638,13 +1638,13 @@ func TestResponsesInputTokensEndpointSpec_ParseBody(t *testing.T) { func TestResponsesInputTokensEndpointSpec_GetTranslator(t *testing.T) { spec := ResponsesInputTokensEndpointSpec{} - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, ModelNameOverride: "override"}) require.NoError(t, err) - _, err = spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaAzureOpenAI, Version: "2025-01-01-preview"}, "override") + _, err = spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAzureOpenAI, Version: "2025-01-01-preview"}, ModelNameOverride: "override"}) require.NoError(t, err) - _, err = spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaAnthropic}, "override") + _, err = spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAnthropic}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "unsupported API schema") } @@ -1905,11 +1905,11 @@ func TestMessagesCountTokensEndpointSpec_GetTranslator(t *testing.T) { {Name: filterapi.APISchemaAWSAnthropic}, {Name: filterapi.APISchemaAnthropic}, } { - translator, err := spec.GetTranslator(schema, "override") + translator, err := spec.GetTranslator(&filterapi.Backend{Schema: schema, ModelNameOverride: "override"}) require.NoError(t, err) require.NotNil(t, translator) } - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "unsupported") } diff --git a/internal/extproc/processor_impl.go b/internal/extproc/processor_impl.go index 6c25990f25..6e9eaf4556 100644 --- a/internal/extproc/processor_impl.go +++ b/internal/extproc/processor_impl.go @@ -722,7 +722,7 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) SetBackend(c } u.parent = rp // Set parent before GetTranslator so it can access rp.eh - u.translator, err = u.parent.eh.GetTranslator(backend.Backend.Schema, u.modelNameOverride) + u.translator, err = u.parent.eh.GetTranslator(backend.Backend) if err != nil { if userFacingErr := internalapi.GetUserFacingError(err); userFacingErr != nil { // The endpoint is not supported for this backend's API schema. That is a mismatch between the diff --git a/internal/filterapi/filterconfig.go b/internal/filterapi/filterconfig.go index f2b7775b6f..32220b079e 100644 --- a/internal/filterapi/filterconfig.go +++ b/internal/filterapi/filterconfig.go @@ -207,6 +207,34 @@ type Backend struct { // HeaderValueFilters filter individual values out of multi-valued request headers before sending // the request to the backend. Optional. HeaderValueFilters []HTTPHeaderValueFilter `json:"headerValueFilters,omitempty"` + + // TranslationHints provides optional model-specific hints to guide request and response + // translation for this model-backend pairing. When nil (or a given field is nil), translators + // fall back to their built-in model-name heuristics. Optional. + TranslationHints *ModelTranslationHints `json:"translationHints,omitempty"` +} + +// ModelTranslationHints provides model-specific hints to guide request and response translation +// for a model-backend pairing. When set, translators use these values instead of model-name +// heuristics. All fields are optional; a nil field falls back to the built-in heuristic for +// backward compatibility. +// +// This mirrors the aigv1b1.ModelTranslationHints CRD type and is serialized into the filter +// config so translators can read it at runtime without Kubernetes access. +type ModelTranslationHints struct { + // MaxOutputTokens is the maximum number of tokens the model can produce in a single response. + // When set, translators use this as the default max_tokens value for APIs that require it + // (e.g., the Anthropic Messages API) when the client request omits max_tokens. + MaxOutputTokens *int64 `json:"maxOutputTokens,omitempty"` + // SupportsResponseJSONSchema indicates whether the model accepts a JSON Schema object in + // response_format (as opposed to only {"type": "json_object"}). + SupportsResponseJSONSchema *bool `json:"supportsResponseJsonSchema,omitempty"` + // SupportsReasoningEffort indicates whether the model accepts a reasoning effort parameter + // (Gemini thinking level / Anthropic output_config.effort). + SupportsReasoningEffort *bool `json:"supportsReasoningEffort,omitempty"` + // SupportsOutputConfig indicates whether the model accepts an Anthropic-style output_config + // field for structured output (e.g., Claude Opus/Sonnet 4.5 and later). + SupportsOutputConfig *bool `json:"supportsOutputConfig,omitempty"` } // BackendAuth corresponds partially to BackendSecurityPolicy in api/v1alpha1/api.go. diff --git a/internal/translator/anthropic_helper.go b/internal/translator/anthropic_helper.go index 77bde83d8e..ccd3856eab 100644 --- a/internal/translator/anthropic_helper.go +++ b/internal/translator/anthropic_helper.go @@ -673,7 +673,13 @@ var gcpOutputConfigModels = []string{ "mythos-preview", // Claude Mythos Preview } -func outputConfigAvailable(apiSchema filterapi.APISchemaName, model internalapi.RequestModel) bool { +// outputConfigAvailable reports whether the model supports Anthropic structured output. +// An explicit backend-scoped hint is authoritative; otherwise the provider-specific +// model-name heuristic preserves backward compatibility. +func outputConfigAvailable(apiSchema filterapi.APISchemaName, model internalapi.RequestModel, hints *filterapi.ModelTranslationHints) bool { + if hints != nil && hints.SupportsOutputConfig != nil { + return *hints.SupportsOutputConfig + } switch apiSchema { case filterapi.APISchemaGCPAnthropic: return modelContainsAny(model, gcpOutputConfigModels) @@ -704,7 +710,13 @@ var effortModels = []string{ "mythos-preview", // Claude Mythos Preview } -func effortAvailable(model internalapi.RequestModel) bool { +// effortAvailable reports whether the model supports the output_config.effort parameter. +// When hints declare SupportsReasoningEffort, that value is authoritative; otherwise it falls back +// to the model-name heuristic for backward compatibility. +func effortAvailable(model internalapi.RequestModel, hints *filterapi.ModelTranslationHints) bool { + if hints != nil && hints.SupportsReasoningEffort != nil { + return *hints.SupportsReasoningEffort + } return modelContainsAny(model, effortModels) } @@ -731,13 +743,16 @@ func mapReasoningEffortToOutputConfigEffort(reasonEffort openai.ReasoningEffort) // into the parameter struct required by the Anthropic SDK. // The apiSchema parameter indicates the backend API schema (e.g., APISchemaAWSAnthropic, // APISchemaGCPAnthropic) and is used to gate backend-specific feature support. -func buildAnthropicParams(openAIReq *openai.ChatCompletionRequest, apiSchema filterapi.APISchemaName, modelNameOverride internalapi.ModelNameOverride) (params *anthropic.MessageNewParams, err error) { +func buildAnthropicParams(openAIReq *openai.ChatCompletionRequest, apiSchema filterapi.APISchemaName, modelNameOverride internalapi.ModelNameOverride, hints *filterapi.ModelTranslationHints) (params *anthropic.MessageNewParams, err error) { // 1. Handle simple parameters. // max_tokens is required by the Anthropic API but optional in the OpenAI API. - // If not set, pass 0 and let the Anthropic API reject the request. + // If the client omits it, default to the model's declared max output tokens when hints provide + // it; otherwise pass 0 and let the Anthropic API reject the request (existing behavior). var maxTokensVal int64 if maxTokens := cmp.Or(openAIReq.MaxCompletionTokens, openAIReq.MaxTokens); maxTokens != nil { maxTokensVal = *maxTokens + } else if hints != nil && hints.MaxOutputTokens != nil { + maxTokensVal = *hints.MaxOutputTokens } // Translate openAI contents to anthropic params. @@ -771,7 +786,7 @@ func buildAnthropicParams(openAIReq *openai.ChatCompletionRequest, apiSchema fil if modelNameOverride != "" { featureCheckModel = modelNameOverride } - if openAIReq.ResponseFormat != nil && openAIReq.ResponseFormat.OfJSONSchema != nil && outputConfigAvailable(apiSchema, featureCheckModel) { + if openAIReq.ResponseFormat != nil && openAIReq.ResponseFormat.OfJSONSchema != nil && outputConfigAvailable(apiSchema, featureCheckModel, hints) { // Validate that the OpenAI JSON schema is an object while retaining its // original bytes. Anthropic's SDK sorts map keys when marshaling, which // would otherwise change the property order seen by Claude. @@ -793,7 +808,7 @@ func buildAnthropicParams(openAIReq *openai.ChatCompletionRequest, apiSchema fil } // Map OpenAI reasoning_effort to Anthropic output_config.effort. - if openAIReq.ReasoningEffort != "" && effortAvailable(featureCheckModel) { + if openAIReq.ReasoningEffort != "" && effortAvailable(featureCheckModel, hints) { effort, effortErr := mapReasoningEffortToOutputConfigEffort(openAIReq.ReasoningEffort) if effortErr != nil { return nil, effortErr diff --git a/internal/translator/anthropic_helper_test.go b/internal/translator/anthropic_helper_test.go index 0afd273fe1..024c50e8f6 100644 --- a/internal/translator/anthropic_helper_test.go +++ b/internal/translator/anthropic_helper_test.go @@ -891,7 +891,7 @@ func TestOutputConfigAvailable(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := outputConfigAvailable(tt.apiSchema, tt.model) + result := outputConfigAvailable(tt.apiSchema, tt.model, nil) require.Equal(t, tt.expected, result) }) } @@ -1299,12 +1299,59 @@ func TestEffortAvailable(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := effortAvailable(tt.model) + result := effortAvailable(tt.model, nil) require.Equal(t, tt.expected, result) }) } } +// TestOutputConfigAvailableWithHints verifies that translation hints override the model-name heuristic. +func TestOutputConfigAvailableWithHints(t *testing.T) { + // Hint true forces support even for a model the heuristic would reject. + require.True(t, outputConfigAvailable(filterapi.APISchemaAWSAnthropic, "claude-3-sonnet", &filterapi.ModelTranslationHints{SupportsOutputConfig: ptr.To(true)})) + // Hint false forces no support even for a model the heuristic would accept. + require.False(t, outputConfigAvailable(filterapi.APISchemaAWSAnthropic, "claude-opus-4-6-20250514", &filterapi.ModelTranslationHints{SupportsOutputConfig: ptr.To(false)})) + // Nil field falls back to the heuristic. + require.True(t, outputConfigAvailable(filterapi.APISchemaAWSAnthropic, "claude-opus-4-6-20250514", &filterapi.ModelTranslationHints{})) +} + +// TestEffortAvailableWithHints verifies that translation hints override the model-name heuristic. +func TestEffortAvailableWithHints(t *testing.T) { + require.True(t, effortAvailable("claude-3-sonnet", &filterapi.ModelTranslationHints{SupportsReasoningEffort: ptr.To(true)})) + require.False(t, effortAvailable("claude-opus-4-6-20250514", &filterapi.ModelTranslationHints{SupportsReasoningEffort: ptr.To(false)})) + require.True(t, effortAvailable("claude-opus-4-6-20250514", &filterapi.ModelTranslationHints{})) +} + +// TestBuildAnthropicParamsMaxTokensFromHints verifies max_tokens defaulting behavior when the client omits it. +func TestBuildAnthropicParamsMaxTokensFromHints(t *testing.T) { + newReq := func() *openai.ChatCompletionRequest { + return &openai.ChatCompletionRequest{ + Model: "claude-opus-4-6", + Messages: []openai.ChatCompletionMessageParamUnion{{OfUser: &openai.ChatCompletionUserMessageParam{Content: openai.StringOrUserRoleContentUnion{Value: "hi"}}}}, + } + } + + t.Run("client omits max_tokens, hint provides default", func(t *testing.T) { + params, err := buildAnthropicParams(newReq(), filterapi.APISchemaAWSAnthropic, "", &filterapi.ModelTranslationHints{MaxOutputTokens: ptr.To(int64(32000))}) + require.NoError(t, err) + require.Equal(t, int64(32000), params.MaxTokens) + }) + + t.Run("client max_tokens takes precedence over hint", func(t *testing.T) { + req := newReq() + req.MaxTokens = ptr.To(int64(100)) + params, err := buildAnthropicParams(req, filterapi.APISchemaAWSAnthropic, "", &filterapi.ModelTranslationHints{MaxOutputTokens: ptr.To(int64(32000))}) + require.NoError(t, err) + require.Equal(t, int64(100), params.MaxTokens) + }) + + t.Run("no hint and no client value keeps existing behavior (0)", func(t *testing.T) { + params, err := buildAnthropicParams(newReq(), filterapi.APISchemaAWSAnthropic, "", nil) + require.NoError(t, err) + require.Equal(t, int64(0), params.MaxTokens) + }) +} + func TestBuildAnthropicParamsWithStructuredOutput(t *testing.T) { tests := []struct { name string @@ -1500,7 +1547,7 @@ func TestBuildAnthropicParamsWithStructuredOutput(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - params, err := buildAnthropicParams(tt.request, tt.apiSchema, "") + params, err := buildAnthropicParams(tt.request, tt.apiSchema, "", nil) if tt.expectErr { require.Error(t, err) @@ -1541,7 +1588,7 @@ func TestBuildAnthropicParamsWithStructuredOutput(t *testing.T) { }, } // The modelNameOverride contains a recognized model identifier. - params, err := buildAnthropicParams(request, filterapi.APISchemaAWSAnthropic, "us.anthropic.claude-sonnet-4-5-20250514-v1:0") + params, err := buildAnthropicParams(request, filterapi.APISchemaAWSAnthropic, "us.anthropic.claude-sonnet-4-5-20250514-v1:0", nil) require.NoError(t, err) require.NotNil(t, params) require.NotNil(t, params.OutputConfig.Format.Schema) @@ -1571,7 +1618,7 @@ func TestBuildAnthropicParamsPreservesStructuredOutputPropertyOrder(t *testing.T }, } - params, err := buildAnthropicParams(request, filterapi.APISchemaAWSAnthropic, "") + params, err := buildAnthropicParams(request, filterapi.APISchemaAWSAnthropic, "", nil) require.NoError(t, err) body, err := json.Marshal(params) @@ -1708,7 +1755,7 @@ func TestBuildAnthropicParamsWithReasoningEffort(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - params, err := buildAnthropicParams(tt.request, filterapi.APISchemaAWSAnthropic, "") + params, err := buildAnthropicParams(tt.request, filterapi.APISchemaAWSAnthropic, "", nil) require.NoError(t, err) require.NotNil(t, params) require.Equal(t, tt.expectedEffort, params.OutputConfig.Effort) @@ -1727,7 +1774,7 @@ func TestBuildAnthropicParamsWithReasoningEffort(t *testing.T) { }}, }, } - _, err := buildAnthropicParams(request, filterapi.APISchemaAWSAnthropic, "") + _, err := buildAnthropicParams(request, filterapi.APISchemaAWSAnthropic, "", nil) require.Error(t, err) require.ErrorIs(t, err, internalapi.ErrInvalidRequestBody) require.Contains(t, err.Error(), "unsupported reasoning effort level") @@ -1746,7 +1793,7 @@ func TestBuildAnthropicParamsWithReasoningEffort(t *testing.T) { }, } // The modelNameOverride contains a recognized model identifier. - params, err := buildAnthropicParams(request, filterapi.APISchemaAWSAnthropic, "us.anthropic.claude-opus-4-5-20250514-v1:0") + params, err := buildAnthropicParams(request, filterapi.APISchemaAWSAnthropic, "us.anthropic.claude-opus-4-5-20250514-v1:0", nil) require.NoError(t, err) require.NotNil(t, params) require.Equal(t, anthropic.OutputConfigEffortHigh, params.OutputConfig.Effort) @@ -1765,7 +1812,7 @@ func TestBuildAnthropicParamsWithReasoningEffort(t *testing.T) { }, } // The modelNameOverride points to an unsupported model. - params, err := buildAnthropicParams(request, filterapi.APISchemaAWSAnthropic, "us.anthropic.claude-3-sonnet-20240229-v1:0") + params, err := buildAnthropicParams(request, filterapi.APISchemaAWSAnthropic, "us.anthropic.claude-3-sonnet-20240229-v1:0", nil) require.NoError(t, err) require.NotNil(t, params) require.Equal(t, anthropic.OutputConfigEffort(""), params.OutputConfig.Effort) diff --git a/internal/translator/gemini_helper.go b/internal/translator/gemini_helper.go index e126bfe4a4..adf1febac7 100644 --- a/internal/translator/gemini_helper.go +++ b/internal/translator/gemini_helper.go @@ -20,6 +20,7 @@ import ( "google.golang.org/genai" "github.com/envoyproxy/ai-gateway/internal/apischema/openai" + "github.com/envoyproxy/ai-gateway/internal/filterapi" "github.com/envoyproxy/ai-gateway/internal/internalapi" "github.com/envoyproxy/ai-gateway/internal/json" ) @@ -54,7 +55,7 @@ const ( // -------------------------------------------------------------. // openAIMessagesToGeminiContents converts OpenAI messages to Gemini Contents and SystemInstruction. -func openAIMessagesToGeminiContents(messages []openai.ChatCompletionMessageParamUnion, requestModel internalapi.RequestModel) ([]genai.Content, *genai.Content, error) { +func openAIMessagesToGeminiContents(messages []openai.ChatCompletionMessageParamUnion, requestModel internalapi.RequestModel, hints *filterapi.ModelTranslationHints) ([]genai.Content, *genai.Content, error) { var gcpContents []genai.Content var systemInstruction *genai.Content knownToolCalls := make(map[string]string) @@ -89,7 +90,7 @@ func openAIMessagesToGeminiContents(messages []openai.ChatCompletionMessageParam } case msgUnion.OfUser != nil: msg := msgUnion.OfUser - parts, err := userMsgToGeminiParts(*msg, requestModel) + parts, err := userMsgToGeminiParts(*msg, requestModel, hints) if err != nil { return nil, nil, fmt.Errorf("invalid user message: %w", err) } @@ -170,7 +171,7 @@ func developerMsgToGeminiParts(msg openai.ChatCompletionDeveloperMessageParam) ( } // userMsgToGeminiParts converts OpenAI user message to Gemini Parts. -func userMsgToGeminiParts(msg openai.ChatCompletionUserMessageParam, requestModel internalapi.RequestModel) ([]*genai.Part, error) { +func userMsgToGeminiParts(msg openai.ChatCompletionUserMessageParam, requestModel internalapi.RequestModel, hints *filterapi.ModelTranslationHints) ([]*genai.Part, error) { var parts []*genai.Part switch contentValue := msg.Content.Value.(type) { case string: @@ -211,7 +212,7 @@ func userMsgToGeminiParts(msg openai.ChatCompletionUserMessageParam, requestMode } // Handle media resolution for both data URI and regular URL cases - if content.OfImageURL.ImageURL.Detail != "" && mediaResolutionAvailable(requestModel) { + if content.OfImageURL.ImageURL.Detail != "" && mediaResolutionAvailable(requestModel, hints) { mediaResolution, err := mapDetailMediaResolution(content.OfImageURL.ImageURL.Detail) if err != nil { return nil, fmt.Errorf("%w: invalid detail", internalapi.ErrInvalidRequestBody) @@ -581,20 +582,35 @@ func openAIToolChoiceToGeminiToolConfig(toolChoice *openai.ChatCompletionToolCho // Check and Update these functions when new Gemini model versions are released // and their feature support changes. -// it only works with models after gemini2.5 according to https://ai.google.dev/gemini-api/docs/structured-output#json-schema, separate it as a small function to make it easier to maintain -func responseJSONSchemaAvailable(requestModel internalapi.RequestModel) bool { +// responseJSONSchemaAvailable reports whether the model accepts a JSON Schema in response_format. +// When hints declare SupportsResponseJSONSchema, that value is authoritative; otherwise it falls back +// to the model-name heuristic (JSON schema only works with Gemini 2.5+ per +// https://ai.google.dev/gemini-api/docs/structured-output#json-schema). +func responseJSONSchemaAvailable(requestModel internalapi.RequestModel, hints *filterapi.ModelTranslationHints) bool { + if hints != nil && hints.SupportsResponseJSONSchema != nil { + return *hints.SupportsResponseJSONSchema + } return strings.Contains(requestModel, "gemini") && (strings.Contains(requestModel, "2.5") || strings.Contains(requestModel, "3")) } // mediaResolutionAvailable checks if the model supports media resolution settings. -// Only Gemini 3.0+ models support this feature for controlling image/video quality. -func mediaResolutionAvailable(requestModel internalapi.RequestModel) bool { +// When hints declare SupportsReasoningEffort, that value is authoritative (media resolution is gated +// by the same Gemini 3.0+ generation as reasoning effort); otherwise it falls back to the model-name +// heuristic. Only Gemini 3.0+ models support this feature for controlling image/video quality. +func mediaResolutionAvailable(requestModel internalapi.RequestModel, hints *filterapi.ModelTranslationHints) bool { + if hints != nil && hints.SupportsReasoningEffort != nil { + return *hints.SupportsReasoningEffort + } return strings.Contains(requestModel, "gemini") && strings.Contains(requestModel, "3") } // reasoningEffortAvailable checks if the model supports reasoning effort. -// Only Gemini 3.0+ models support this feature for controlling reasoning depth. -func reasoningEffortAvailable(requestModel internalapi.RequestModel) bool { +// When hints declare SupportsReasoningEffort, that value is authoritative; otherwise it falls back to +// the model-name heuristic. Only Gemini 3.0+ models support this feature for controlling reasoning depth. +func reasoningEffortAvailable(requestModel internalapi.RequestModel, hints *filterapi.ModelTranslationHints) bool { + if hints != nil && hints.SupportsReasoningEffort != nil { + return *hints.SupportsReasoningEffort + } return strings.Contains(requestModel, "gemini") && strings.Contains(requestModel, "3") } @@ -637,7 +653,7 @@ func mapReasoningEffortToThinkingLevel(reasonEffort openai.ReasoningEffort, mode } // openAIReqToGeminiGenerationConfig converts OpenAI request to Gemini GenerationConfig. -func openAIReqToGeminiGenerationConfig(openAIReq *openai.ChatCompletionRequest, requestModel internalapi.RequestModel) (*genai.GenerationConfig, geminiResponseMode, error) { +func openAIReqToGeminiGenerationConfig(openAIReq *openai.ChatCompletionRequest, requestModel internalapi.RequestModel, hints *filterapi.ModelTranslationHints) (*genai.GenerationConfig, geminiResponseMode, error) { responseMode := responseModeNone gc := &genai.GenerationConfig{} if openAIReq.Temperature != nil { @@ -683,7 +699,7 @@ func openAIReqToGeminiGenerationConfig(openAIReq *openai.ChatCompletionRequest, responseMode = responseModeJSON - if responseJSONSchemaAvailable(requestModel) { + if responseJSONSchemaAvailable(requestModel, hints) { gc.ResponseJsonSchema = schemaMap } else { convertedSchema, err := jsonSchemaToGemini(schemaMap) @@ -744,7 +760,7 @@ func openAIReqToGeminiGenerationConfig(openAIReq *openai.ChatCompletionRequest, gc.ResponseMIMEType = mimeTypeApplicationJSON gc.ResponseJsonSchema = guidedJSON } - if openAIReq.ReasoningEffort != "" && reasoningEffortAvailable(requestModel) { + if openAIReq.ReasoningEffort != "" && reasoningEffortAvailable(requestModel, hints) { thinkLevel, err := mapReasoningEffortToThinkingLevel(openAIReq.ReasoningEffort, requestModel) if err != nil { return nil, responseMode, fmt.Errorf("invalid reasoning effort: %w", err) diff --git a/internal/translator/gemini_helper_test.go b/internal/translator/gemini_helper_test.go index 529efc726a..5246fb7c92 100644 --- a/internal/translator/gemini_helper_test.go +++ b/internal/translator/gemini_helper_test.go @@ -20,10 +20,34 @@ import ( "k8s.io/utils/ptr" "github.com/envoyproxy/ai-gateway/internal/apischema/openai" + "github.com/envoyproxy/ai-gateway/internal/filterapi" "github.com/envoyproxy/ai-gateway/internal/internalapi" "github.com/envoyproxy/ai-gateway/internal/json" ) +// TestGeminiFeatureGatesWithHints verifies that translation hints override the model-name heuristics. +func TestGeminiFeatureGatesWithHints(t *testing.T) { + t.Run("responseJSONSchemaAvailable", func(t *testing.T) { + // Hint true forces support even for a model the heuristic would reject (e.g. gemini-4). + require.True(t, responseJSONSchemaAvailable("gemini-4-pro", &filterapi.ModelTranslationHints{SupportsResponseJSONSchema: ptr.To(true)})) + // Hint false forces no support even for a model the heuristic would accept. + require.False(t, responseJSONSchemaAvailable("gemini-2.5-pro", &filterapi.ModelTranslationHints{SupportsResponseJSONSchema: ptr.To(false)})) + // Nil field falls back to the heuristic. + require.True(t, responseJSONSchemaAvailable("gemini-2.5-pro", &filterapi.ModelTranslationHints{})) + require.False(t, responseJSONSchemaAvailable("gemini-4-pro", nil)) + }) + t.Run("reasoningEffortAvailable", func(t *testing.T) { + require.True(t, reasoningEffortAvailable("gemini-4-pro", &filterapi.ModelTranslationHints{SupportsReasoningEffort: ptr.To(true)})) + require.False(t, reasoningEffortAvailable("gemini-3-pro", &filterapi.ModelTranslationHints{SupportsReasoningEffort: ptr.To(false)})) + require.True(t, reasoningEffortAvailable("gemini-3-pro", nil)) + }) + t.Run("mediaResolutionAvailable", func(t *testing.T) { + require.True(t, mediaResolutionAvailable("gemini-4-pro", &filterapi.ModelTranslationHints{SupportsReasoningEffort: ptr.To(true)})) + require.False(t, mediaResolutionAvailable("gemini-3-pro", &filterapi.ModelTranslationHints{SupportsReasoningEffort: ptr.To(false)})) + require.True(t, mediaResolutionAvailable("gemini-3-pro", nil)) + }) +} + func TestOpenAIMessagesToGeminiContents(t *testing.T) { tests := []struct { name string @@ -124,7 +148,7 @@ func TestOpenAIMessagesToGeminiContents(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - contents, systemInstruction, err := openAIMessagesToGeminiContents(tc.messages, "gemini-3-pro") + contents, systemInstruction, err := openAIMessagesToGeminiContents(tc.messages, "gemini-3-pro", nil) if tc.expectedErrorMsg != "" || err != nil { require.Error(t, err) @@ -1227,7 +1251,7 @@ func TestUserMsgToGeminiParts(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - parts, err := userMsgToGeminiParts(tc.msg, "gemini-3-pro") + parts, err := userMsgToGeminiParts(tc.msg, "gemini-3-pro", nil) if tc.expectedErrMsg != "" || err != nil { require.Error(t, err) @@ -1589,7 +1613,7 @@ func TestOpenAIReqToGeminiGenerationConfig(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - got, responseMode, err := openAIReqToGeminiGenerationConfig(tc.input, tc.requestModel) + got, responseMode, err := openAIReqToGeminiGenerationConfig(tc.input, tc.requestModel, nil) if tc.expectedErrMsg != "" { require.ErrorContains(t, err, tc.expectedErrMsg) } else { @@ -3336,7 +3360,7 @@ func TestOpenAIReqToGeminiGenerationConfigWithJsonSchemaToGemini(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - got, responseMode, err := openAIReqToGeminiGenerationConfig(tc.input, tc.requestModel) + got, responseMode, err := openAIReqToGeminiGenerationConfig(tc.input, tc.requestModel, nil) if tc.expectedErrMsg != "" { require.ErrorContains(t, err, tc.expectedErrMsg) require.ErrorIs(t, err, internalapi.ErrInvalidRequestBody) diff --git a/internal/translator/openai_awsanthropic.go b/internal/translator/openai_awsanthropic.go index 2b5d53f5ea..b948741705 100644 --- a/internal/translator/openai_awsanthropic.go +++ b/internal/translator/openai_awsanthropic.go @@ -33,10 +33,11 @@ const BedrockDefaultVersion = "bedrock-2023-05-31" // NewChatCompletionOpenAIToAWSAnthropicTranslator implements [Factory] for OpenAI to AWS Anthropic translation. // This translator converts OpenAI ChatCompletion API requests to AWS Anthropic API format. -func NewChatCompletionOpenAIToAWSAnthropicTranslator(apiVersion string, modelNameOverride internalapi.ModelNameOverride) OpenAIChatCompletionTranslator { +func NewChatCompletionOpenAIToAWSAnthropicTranslator(apiVersion string, modelNameOverride internalapi.ModelNameOverride, hints *filterapi.ModelTranslationHints) OpenAIChatCompletionTranslator { return &openAIToAWSAnthropicTranslatorV1ChatCompletion{ apiVersion: apiVersion, modelNameOverride: modelNameOverride, + hints: hints, } } @@ -46,6 +47,7 @@ func NewChatCompletionOpenAIToAWSAnthropicTranslator(apiVersion string, modelNam type openAIToAWSAnthropicTranslatorV1ChatCompletion struct { apiVersion string modelNameOverride internalapi.ModelNameOverride + hints *filterapi.ModelTranslationHints streamParser *anthropicStreamParser requestModel internalapi.RequestModel bufferedBody []byte @@ -71,7 +73,7 @@ func (o *openAIToAWSAnthropicTranslatorV1ChatCompletion) RequestBody(_ []byte, o o.streamParser = newAnthropicStreamParser(o.requestModel) } - params, err := buildAnthropicParams(openAIReq, filterapi.APISchemaAWSAnthropic, o.modelNameOverride) + params, err := buildAnthropicParams(openAIReq, filterapi.APISchemaAWSAnthropic, o.modelNameOverride, o.hints) if err != nil { return } diff --git a/internal/translator/openai_awsanthropic_test.go b/internal/translator/openai_awsanthropic_test.go index 44762b448f..96f9e29e25 100644 --- a/internal/translator/openai_awsanthropic_test.go +++ b/internal/translator/openai_awsanthropic_test.go @@ -111,7 +111,7 @@ func wrapAnthropicSSEInEventStream(sseData string) ([]byte, error) { // AWS Anthropic uses deterministic model mapping without virtualization func TestResponseModel_AWSAnthropic(t *testing.T) { modelName := "anthropic.claude-sonnet-4-20250514-v1:0" - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", modelName) + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", modelName, nil) // Initialize translator with the model req := &openai.ChatCompletionRequest{ @@ -179,7 +179,7 @@ func TestOpenAIToAWSAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T } t.Run("AWS Bedrock InvokeModel Values Configured Correctly", func(t *testing.T) { - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "", nil) hm, body, err := translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) require.NotNil(t, hm) @@ -202,7 +202,7 @@ func TestOpenAIToAWSAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T t.Run("Model Name Override", func(t *testing.T) { overrideModelName := "anthropic.claude-3-haiku-20240307-v1:0" // Instantiate the translator with the model name override. - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", overrideModelName) + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", overrideModelName, nil) // Call RequestBody with the original request, which has a different model name. hm, _, err := translator.RequestBody(nil, openAIReq, false) @@ -218,7 +218,7 @@ func TestOpenAIToAWSAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T t.Run("Model Name with ARN (URL encoding)", func(t *testing.T) { arnModelName := "arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-3-opus-20240229-v1:0" - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", arnModelName) + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", arnModelName, nil) hm, _, err := translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -240,7 +240,7 @@ func TestOpenAIToAWSAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T MaxTokens: ptr.To(int64(100)), Stream: true, } - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "", nil) hm, body, err := translator.RequestBody(nil, streamReq, false) require.NoError(t, err) require.NotNil(t, hm) @@ -261,7 +261,7 @@ func TestOpenAIToAWSAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T t.Run("API Version Override", func(t *testing.T) { customAPIVersion := "bedrock-2024-01-01" // Instantiate the translator with the custom API version. - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator(customAPIVersion, "") + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator(customAPIVersion, "", nil) // Call RequestBody with a standard request. _, body, err := translator.RequestBody(nil, openAIReq, false) @@ -279,7 +279,7 @@ func TestOpenAIToAWSAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T MaxTokens: ptr.To(int64(100)), Temperature: ptr.To(2.5), } - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "", nil) _, _, err := translator.RequestBody(nil, invalidTempReq, false) require.Error(t, err) require.Contains(t, err.Error(), fmt.Sprintf(tempNotSupportedError, *invalidTempReq.Temperature)) @@ -290,7 +290,7 @@ func TestOpenAIToAWSAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T Model: "anthropic.claude-3-opus-20240229-v1:0", Messages: []openai.ChatCompletionMessageParamUnion{}, } - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, missingTokensReq, false) require.NoError(t, err) require.Equal(t, int64(0), gjson.GetBytes(body, "max_tokens").Int()) @@ -299,7 +299,7 @@ func TestOpenAIToAWSAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T func TestOpenAIToAWSAnthropicTranslatorV1ChatCompletion_ResponseBody(t *testing.T) { t.Run("invalid json body", func(t *testing.T) { - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "", nil) _, _, _, _, err := translator.ResponseBody(map[string]string{statusHeaderName: "200"}, bytes.NewBufferString("invalid json"), true, nil) require.Error(t, err) require.Contains(t, err.Error(), "failed to unmarshal body") @@ -400,7 +400,7 @@ func TestOpenAIToAWSAnthropicTranslatorV1ChatCompletion_ResponseBody(t *testing. body, err := json.Marshal(tt.inputResponse) require.NoError(t, err, "Test setup failed: could not marshal input struct") - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "", nil) hm, body, usedToken, _, err := translator.ResponseBody(tt.respHeaders, bytes.NewBuffer(body), true, nil) require.NoError(t, err, "Translator returned an unexpected internal error") @@ -545,7 +545,7 @@ data: {"type": "message_stop"} MaxTokens: new(int64), } - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "").(*openAIToAWSAnthropicTranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "", nil).(*openAIToAWSAnthropicTranslatorV1ChatCompletion) _, _, err = translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -598,7 +598,7 @@ data: {"type": "message_stop"} Model: "test-model", MaxTokens: new(int64), } - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "").(*openAIToAWSAnthropicTranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "", nil).(*openAIToAWSAnthropicTranslatorV1ChatCompletion) _, _, err = translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -650,7 +650,7 @@ data: {"type": "message_stop"} require.NoError(t, err) openAIReq := &openai.ChatCompletionRequest{Stream: true, Model: "test-model", MaxTokens: new(int64)} - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "").(*openAIToAWSAnthropicTranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "", nil).(*openAIToAWSAnthropicTranslatorV1ChatCompletion) _, _, err = translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -705,7 +705,7 @@ data: {"type":"message_stop"} require.NoError(t, err) openAIReq := &openai.ChatCompletionRequest{Stream: true, Model: "test-model", MaxTokens: new(int64)} - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "").(*openAIToAWSAnthropicTranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "", nil).(*openAIToAWSAnthropicTranslatorV1ChatCompletion) _, _, err = translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -728,7 +728,7 @@ func TestAWSAnthropicStreamParser_ErrorHandling(t *testing.T) { require.NoError(t, err) openAIReq := &openai.ChatCompletionRequest{Stream: true, Model: "test-model", MaxTokens: new(int64)} - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "").(*openAIToAWSAnthropicTranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "", nil).(*openAIToAWSAnthropicTranslatorV1ChatCompletion) _, _, err = translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -801,7 +801,7 @@ func TestOpenAIToAWSAnthropicTranslator_EdgeCases(t *testing.T) { t.Run("response with model field from API", func(t *testing.T) { // AWS Anthropic may return model field in response modelName := "custom-override-model" - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", modelName) + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", modelName, nil) req := &openai.ChatCompletionRequest{ Model: "original-model", @@ -838,7 +838,7 @@ func TestOpenAIToAWSAnthropicTranslator_EdgeCases(t *testing.T) { t.Run("response without model field", func(t *testing.T) { // AWS Anthropic typically doesn't return model field modelName := "anthropic.claude-3-haiku-20240307-v1:0" - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", modelName) + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", modelName, nil) req := &openai.ChatCompletionRequest{ Model: "original-model", diff --git a/internal/translator/openai_gcpanthropic.go b/internal/translator/openai_gcpanthropic.go index 39bf39bb2a..b22fc1222a 100644 --- a/internal/translator/openai_gcpanthropic.go +++ b/internal/translator/openai_gcpanthropic.go @@ -32,10 +32,11 @@ const ( // NewChatCompletionOpenAIToGCPAnthropicTranslator implements [Factory] for OpenAI to GCP Anthropic translation. // This translator converts OpenAI ChatCompletion API requests to GCP Anthropic API format. -func NewChatCompletionOpenAIToGCPAnthropicTranslator(apiVersion string, modelNameOverride internalapi.ModelNameOverride) OpenAIChatCompletionTranslator { +func NewChatCompletionOpenAIToGCPAnthropicTranslator(apiVersion string, modelNameOverride internalapi.ModelNameOverride, hints *filterapi.ModelTranslationHints) OpenAIChatCompletionTranslator { return &openAIToGCPAnthropicTranslatorV1ChatCompletion{ apiVersion: apiVersion, modelNameOverride: modelNameOverride, + hints: hints, } } @@ -45,6 +46,7 @@ func NewChatCompletionOpenAIToGCPAnthropicTranslator(apiVersion string, modelNam type openAIToGCPAnthropicTranslatorV1ChatCompletion struct { apiVersion string modelNameOverride internalapi.ModelNameOverride + hints *filterapi.ModelTranslationHints streamParser *anthropicStreamParser requestModel internalapi.RequestModel // Redaction configuration for debug logging @@ -57,7 +59,7 @@ type openAIToGCPAnthropicTranslatorV1ChatCompletion struct { func (o *openAIToGCPAnthropicTranslatorV1ChatCompletion) RequestBody(_ []byte, openAIReq *openai.ChatCompletionRequest, _ bool) ( newHeaders []internalapi.Header, newBody []byte, err error, ) { - params, err := buildAnthropicParams(openAIReq, filterapi.APISchemaGCPAnthropic, o.modelNameOverride) + params, err := buildAnthropicParams(openAIReq, filterapi.APISchemaGCPAnthropic, o.modelNameOverride, o.hints) if err != nil { return } diff --git a/internal/translator/openai_gcpanthropic_test.go b/internal/translator/openai_gcpanthropic_test.go index ce2d7c7e59..bb4360d72c 100644 --- a/internal/translator/openai_gcpanthropic_test.go +++ b/internal/translator/openai_gcpanthropic_test.go @@ -42,7 +42,7 @@ const ( // GCP Anthropic uses deterministic model mapping without virtualization func TestResponseModel_GCPAnthropic(t *testing.T) { modelName := "claude-sonnet-4@20250514" - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", modelName) + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", modelName, nil) // Initialize translator with the model req := &openai.ChatCompletionRequest{ @@ -109,7 +109,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T Temperature: ptr.To(0.7), } t.Run("Vertex Values Configured Correctly", func(t *testing.T) { - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) hm, body, err := translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) require.NotNil(t, hm) @@ -133,7 +133,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T t.Run("Model Name Override", func(t *testing.T) { overrideModelName := "claude-3" // Instantiate the translator with the model name override. - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", overrideModelName) + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", overrideModelName, nil) // Call RequestBody with the original request, which has a different model name. hm, _, err := translator.RequestBody(nil, openAIReq, false) @@ -169,7 +169,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T }, }, } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, imageReq, false) require.NoError(t, err) @@ -193,7 +193,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T }, MaxTokens: ptr.To(int64(100)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, multiSystemReq, false) require.NoError(t, err) @@ -209,7 +209,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T MaxTokens: ptr.To(int64(100)), Stream: true, } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) hm, body, err := translator.RequestBody(nil, streamReq, false) require.NoError(t, err) require.NotNil(t, hm) @@ -234,7 +234,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T OfStringArray: []string{"stop1", "stop2"}, }, } - messageParam, err := buildAnthropicParams(openaiRequest, filterapi.APISchemaGCPAnthropic, "") + messageParam, err := buildAnthropicParams(openaiRequest, filterapi.APISchemaGCPAnthropic, "", nil) require.NoError(t, err) require.Equal(t, int64(100), messageParam.MaxTokens) require.Equal(t, "0.1", messageParam.TopP.String()) @@ -253,7 +253,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T OfString: openaigo.Opt[string]("stop1"), }, } - messageParam, err := buildAnthropicParams(openaiRequest, filterapi.APISchemaGCPAnthropic, "") + messageParam, err := buildAnthropicParams(openaiRequest, filterapi.APISchemaGCPAnthropic, "", nil) require.NoError(t, err) require.Equal(t, int64(100), messageParam.MaxTokens) require.Equal(t, "0.1", messageParam.TopP.String()) @@ -268,7 +268,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T MaxTokens: ptr.To(int64(100)), Temperature: ptr.To(2.5), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, _, err := translator.RequestBody(nil, invalidTempReq, false) require.ErrorIs(t, err, internalapi.ErrInvalidRequestBody) }) @@ -280,7 +280,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T MaxTokens: ptr.To(int64(100)), Temperature: ptr.To(-2.5), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, _, err := translator.RequestBody(nil, invalidTempReq, false) require.ErrorIs(t, err, internalapi.ErrInvalidRequestBody) }) @@ -290,7 +290,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T Model: claudeTestModel, Messages: []openai.ChatCompletionMessageParamUnion{}, } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, missingTokensReq, false) require.NoError(t, err) require.Equal(t, int64(0), gjson.GetBytes(body, "max_tokens").Int()) @@ -298,7 +298,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T t.Run("API Version Override", func(t *testing.T) { customAPIVersion := "bedrock-2023-05-31" // Instantiate the translator with the custom API version. - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator(customAPIVersion, "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator(customAPIVersion, "", nil) // Call RequestBody with a standard request. _, body, err := translator.RequestBody(nil, openAIReq, false) @@ -322,7 +322,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T }, }, } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, thinkingReq, false) require.NoError(t, err) require.NotNil(t, body) @@ -345,7 +345,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T }, }, } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, thinkingReq, false) require.NoError(t, err) require.NotNil(t, body) @@ -370,7 +370,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T }, }, } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, thinkingReq, false) require.NoError(t, err) require.NotNil(t, body) @@ -393,7 +393,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T }, }, } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, thinkingReq, false) require.NoError(t, err) require.NotNil(t, body) @@ -408,7 +408,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_ResponseBody(t *testing.T) { t.Run("invalid json body", func(t *testing.T) { - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, _, _, _, err := translator.ResponseBody(map[string]string{statusHeaderName: "200"}, bytes.NewBufferString("invalid json"), true, nil) require.Error(t, err) require.Contains(t, err.Error(), "failed to unmarshal body") @@ -636,7 +636,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_ResponseBody(t *testing. body, err := json.Marshal(tt.inputResponse) require.NoError(t, err, "Test setup failed: could not marshal input struct") - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) hm, body, usedToken, _, err := translator.ResponseBody(tt.respHeaders, bytes.NewBuffer(body), true, nil) require.NoError(t, err, "Translator returned an unexpected internal error") @@ -1341,7 +1341,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_Cache(t *testing.T) { MaxTokens: ptr.To(int64(100)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -1438,7 +1438,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_Cache(t *testing.T) { MaxTokens: ptr.To(int64(10)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, req, false) require.NoError(t, err) @@ -1484,7 +1484,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_Cache(t *testing.T) { MaxTokens: ptr.To(int64(50)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, req, false) require.NoError(t, err) @@ -1528,7 +1528,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_Cache(t *testing.T) { MaxTokens: ptr.To(int64(50)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, req, false) require.NoError(t, err) @@ -1562,7 +1562,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_Cache(t *testing.T) { MaxTokens: ptr.To(int64(100)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -1602,7 +1602,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_Cache(t *testing.T) { MaxTokens: ptr.To(int64(100)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -1647,7 +1647,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_Cache(t *testing.T) { MaxTokens: ptr.To(int64(100)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -1691,7 +1691,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_Cache(t *testing.T) { MaxTokens: ptr.To(int64(100)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -1733,7 +1733,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_Cache(t *testing.T) { MaxTokens: ptr.To(int64(100)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -1770,7 +1770,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_Cache(t *testing.T) { MaxTokens: ptr.To(int64(100)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -1824,7 +1824,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_Cache(t *testing.T) { MaxTokens: ptr.To(int64(100)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -1842,7 +1842,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_Cache(t *testing.T) { } func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_SetRedactionConfig(t *testing.T) { - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "").(*openAIToGCPAnthropicTranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil).(*openAIToGCPAnthropicTranslatorV1ChatCompletion) logger := slog.New(slog.NewTextHandler(io.Discard, nil)) translator.SetRedactionConfig(true, true, logger) @@ -1853,7 +1853,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_SetRedactionConfig(t *te } func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RedactBody(t *testing.T) { - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "").(*openAIToGCPAnthropicTranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil).(*openAIToGCPAnthropicTranslatorV1ChatCompletion) t.Run("nil response returns nil", func(t *testing.T) { result := translator.RedactBody(nil) @@ -1991,7 +1991,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_ResponseHeaders(t *testi Model: "test-model", MaxTokens: ptr.To(int64(100)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "").(*openAIToGCPAnthropicTranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil).(*openAIToGCPAnthropicTranslatorV1ChatCompletion) // Initialize the stream parser by calling RequestBody with streaming request _, _, err := translator.RequestBody(nil, openAIReq, false) @@ -2011,7 +2011,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_ResponseHeaders(t *testi Model: "test-model", MaxTokens: ptr.To(int64(100)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "").(*openAIToGCPAnthropicTranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil).(*openAIToGCPAnthropicTranslatorV1ChatCompletion) // Initialize without streaming _, _, err := translator.RequestBody(nil, openAIReq, false) @@ -2029,7 +2029,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_ResponseBody_WithDebugLo var logBuf bytes.Buffer logger := slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelDebug})) - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "").(*openAIToGCPAnthropicTranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil).(*openAIToGCPAnthropicTranslatorV1ChatCompletion) translator.SetRedactionConfig(true, true, logger) // Initialize translator with the model @@ -2091,7 +2091,7 @@ func (m *mockSpan) EndSpanOnError(_ int, _ []byte) {} func (m *mockSpan) EndSpan() {} func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_ResponseBody_WithSpanRecording(t *testing.T) { - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "").(*openAIToGCPAnthropicTranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil).(*openAIToGCPAnthropicTranslatorV1ChatCompletion) // Initialize translator with the model req := &openai.ChatCompletionRequest{ diff --git a/internal/translator/openai_gcpvertexai.go b/internal/translator/openai_gcpvertexai.go index 529f265ab8..37028c3493 100644 --- a/internal/translator/openai_gcpvertexai.go +++ b/internal/translator/openai_gcpvertexai.go @@ -20,6 +20,7 @@ import ( "github.com/envoyproxy/ai-gateway/internal/apischema/gcp" "github.com/envoyproxy/ai-gateway/internal/apischema/openai" + "github.com/envoyproxy/ai-gateway/internal/filterapi" "github.com/envoyproxy/ai-gateway/internal/internalapi" "github.com/envoyproxy/ai-gateway/internal/json" "github.com/envoyproxy/ai-gateway/internal/metrics" @@ -67,8 +68,8 @@ type gcpVertexAIErrorDetails struct { // NewChatCompletionOpenAIToGCPVertexAITranslator implements [Factory] for OpenAI to GCP Gemini translation. // This translator converts OpenAI ChatCompletion API requests to GCP Gemini API format. -func NewChatCompletionOpenAIToGCPVertexAITranslator(modelNameOverride internalapi.ModelNameOverride) OpenAIChatCompletionTranslator { - return &openAIToGCPVertexAITranslatorV1ChatCompletion{modelNameOverride: modelNameOverride, toolCallIndex: int64(0)} +func NewChatCompletionOpenAIToGCPVertexAITranslator(modelNameOverride internalapi.ModelNameOverride, hints *filterapi.ModelTranslationHints) OpenAIChatCompletionTranslator { + return &openAIToGCPVertexAITranslatorV1ChatCompletion{modelNameOverride: modelNameOverride, hints: hints, toolCallIndex: int64(0)} } // openAIToGCPVertexAITranslatorV1ChatCompletion translates OpenAI Chat Completions API to GCP Vertex AI Gemini API. @@ -77,6 +78,7 @@ func NewChatCompletionOpenAIToGCPVertexAITranslator(modelNameOverride internalap type openAIToGCPVertexAITranslatorV1ChatCompletion struct { responseMode geminiResponseMode modelNameOverride internalapi.ModelNameOverride + hints *filterapi.ModelTranslationHints stream bool // Track if this is a streaming request. streamDelimiter []byte bufferedBody []byte // Buffer for incomplete JSON chunks. @@ -518,13 +520,13 @@ func getGenerationConfigThinkingConfig(tu *openai.ThinkingUnion) *genai.Thinking // openAIMessageToGeminiMessage converts an OpenAI ChatCompletionRequest to a GCP Gemini GenerateContentRequest. func (o *openAIToGCPVertexAITranslatorV1ChatCompletion) openAIMessageToGeminiMessage(openAIReq *openai.ChatCompletionRequest, requestModel internalapi.RequestModel) (*gcp.GenerateContentRequest, error) { // Convert OpenAI messages to Gemini Contents and SystemInstruction. - contents, systemInstruction, err := openAIMessagesToGeminiContents(openAIReq.Messages, requestModel) + contents, systemInstruction, err := openAIMessagesToGeminiContents(openAIReq.Messages, requestModel, o.hints) if err != nil { return nil, err } // Some models support only partialJSONSchema. - parametersJSONSchemaAvailable := responseJSONSchemaAvailable(requestModel) + parametersJSONSchemaAvailable := responseJSONSchemaAvailable(requestModel, o.hints) // Convert OpenAI tools to Gemini tools. tools, err := openAIToolsToGeminiTools(openAIReq.Tools, parametersJSONSchemaAvailable) if err != nil { @@ -538,7 +540,7 @@ func (o *openAIToGCPVertexAITranslatorV1ChatCompletion) openAIMessageToGeminiMes } // Convert generation config. - generationConfig, responseMode, err := openAIReqToGeminiGenerationConfig(openAIReq, requestModel) + generationConfig, responseMode, err := openAIReqToGeminiGenerationConfig(openAIReq, requestModel, o.hints) if err != nil { return nil, fmt.Errorf("invalid generation configs: %w", err) } @@ -577,7 +579,7 @@ func (o *openAIToGCPVertexAITranslatorV1ChatCompletion) applyVendorSpecificField if gcr.GenerationConfig == nil { gcr.GenerationConfig = &genai.GenerationConfig{} } - if vendorGenConfig.MediaResolution != "" && mediaResolutionAvailable(requestModel) { + if vendorGenConfig.MediaResolution != "" && mediaResolutionAvailable(requestModel, o.hints) { gcr.GenerationConfig.MediaResolution = vendorGenConfig.MediaResolution } } diff --git a/internal/translator/openai_gcpvertexai_embeddings.go b/internal/translator/openai_gcpvertexai_embeddings.go index 4badee4049..460e879636 100644 --- a/internal/translator/openai_gcpvertexai_embeddings.go +++ b/internal/translator/openai_gcpvertexai_embeddings.go @@ -198,7 +198,8 @@ func collectPartsFromMessages(messages []openai.ChatCompletionMessageParamUnion, if msg.OfUser == nil { continue } - msgParts, err := userMsgToGeminiParts(*msg.OfUser, requestModel) + // Embeddings do not carry translation hints; pass nil to use the model-name heuristic. + msgParts, err := userMsgToGeminiParts(*msg.OfUser, requestModel, nil) if err != nil { return nil, err } diff --git a/internal/translator/openai_gcpvertexai_test.go b/internal/translator/openai_gcpvertexai_test.go index 439c82d219..a70daeecf1 100644 --- a/internal/translator/openai_gcpvertexai_test.go +++ b/internal/translator/openai_gcpvertexai_test.go @@ -32,7 +32,7 @@ import ( // GCP Vertex AI uses deterministic model mapping without virtualization func TestResponseModel_GCPVertexAIStreaming(t *testing.T) { modelName := "gemini-1.5-pro-002" - translator := NewChatCompletionOpenAIToGCPVertexAITranslator(modelName).(*openAIToGCPVertexAITranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToGCPVertexAITranslator(modelName, nil).(*openAIToGCPVertexAITranslatorV1ChatCompletion) // Initialize translator with streaming request req := &openai.ChatCompletionRequest{ @@ -784,7 +784,7 @@ func TestOpenAIToGCPVertexAITranslatorV1ChatCompletion_RequestBody(t *testing.T) for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - translator := NewChatCompletionOpenAIToGCPVertexAITranslator(tc.modelNameOverride) + translator := NewChatCompletionOpenAIToGCPVertexAITranslator(tc.modelNameOverride, nil) headerMut, bodyMut, err := translator.RequestBody(nil, &tc.input, tc.onRetry) if tc.wantError { assert.Error(t, err) @@ -839,7 +839,7 @@ func TestOpenAIToGCPVertexAITranslatorV1ChatCompletion_ResponseHeaders(t *testin for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - translator := NewChatCompletionOpenAIToGCPVertexAITranslator(tc.modelName) + translator := NewChatCompletionOpenAIToGCPVertexAITranslator(tc.modelName, nil) _, err := translator.ResponseHeaders(tc.headers) if tc.wantError { assert.Error(t, err) @@ -1652,7 +1652,7 @@ func TestExtractToolCallsFromGeminiPartsStream(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - o := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001").(*openAIToGCPVertexAITranslatorV1ChatCompletion) + o := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001", nil).(*openAIToGCPVertexAITranslatorV1ChatCompletion) calls, _, err := o.extractToolCallsFromGeminiPartsStream(toolCalls, tt.input, json.MarshalForDeterministicTesting) if tt.wantErr { @@ -1687,7 +1687,7 @@ func TestExtractToolCallsStreamVsNonStream(t *testing.T) { }, }, } - o := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001").(*openAIToGCPVertexAITranslatorV1ChatCompletion) + o := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001", nil).(*openAIToGCPVertexAITranslatorV1ChatCompletion) // Get results from both functions streamCalls, _, err := o.extractToolCallsFromGeminiPartsStream(toolCallsStream, parts, json.MarshalForDeterministicTesting) @@ -1752,7 +1752,7 @@ func TestExtractToolCallsStreamIndexing(t *testing.T) { }, }, } - o := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001").(*openAIToGCPVertexAITranslatorV1ChatCompletion) + o := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001", nil).(*openAIToGCPVertexAITranslatorV1ChatCompletion) calls, _, err := o.extractToolCallsFromGeminiPartsStream(toolCalls, parts, json.MarshalForDeterministicTesting) require.NoError(t, err) @@ -1785,7 +1785,7 @@ func TestExtractToolCallsStreamIndexing(t *testing.T) { } func TestOpenAIToGCPVertexAITranslatorV1ChatCompletion_StreamingParallelToolIndex(t *testing.T) { - translator := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001").(*openAIToGCPVertexAITranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001", nil).(*openAIToGCPVertexAITranslatorV1ChatCompletion) // Mock multiple GCP streaming response with parallel tool calls gcpToolCallsChunk := `data: { "candidates": [ @@ -1893,7 +1893,7 @@ data: {"candidates": [ // TestOpenAIToGCPVertexAITranslatorV1ChatCompletion_StreamingToolCallWithSignature tests that // streaming tool calls with thought signatures are correctly translated. func TestOpenAIToGCPVertexAITranslatorV1ChatCompletion_StreamingToolCallWithSignature(t *testing.T) { - translator := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001").(*openAIToGCPVertexAITranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001", nil).(*openAIToGCPVertexAITranslatorV1ChatCompletion) // GCP streaming response with thinking followed by tool call with signature gcpStreamingChunk := `data: {"candidates":[{"content":{"parts":[{"text":"let me think about this.", "thought": true}]}}]} @@ -1954,7 +1954,7 @@ func TestOpenAIToGCPVertexAITranslatorV1ChatCompletion_StreamingToolCallSplitFin // must still be "tool_calls", not "stop". (Older Gemini models carried both in // a single chunk; that case is covered by // TestOpenAIToGCPVertexAITranslatorV1ChatCompletion_StreamingToolCallWithSignature.) - translator := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-3.5-flash").(*openAIToGCPVertexAITranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-3.5-flash", nil).(*openAIToGCPVertexAITranslatorV1ChatCompletion) gcpStreamingChunk := `data: {"candidates":[{"content":{"role":"model","parts":[{"functionCall":{"name":"get_weather","args":{"location":"Paris"}},"thoughtSignature":"dG9vbGNhbGxzaWduYXR1cmU="}]}}],"usageMetadata":{"trafficType":"ON_DEMAND"}} @@ -2000,7 +2000,7 @@ data: {"candidates":[{"content":{"role":"model","parts":[{"text":""}]},"finishRe } func TestOpenAIToGCPVertexAITranslatorV1ChatCompletion_StreamingEndOfStream(t *testing.T) { - translator := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001").(*openAIToGCPVertexAITranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001", nil).(*openAIToGCPVertexAITranslatorV1ChatCompletion) // Test end of stream marker. _, body, _, _, err := translator.handleStreamingResponse( @@ -2492,7 +2492,7 @@ Details: [ for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - translator := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001").(*openAIToGCPVertexAITranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001", nil).(*openAIToGCPVertexAITranslatorV1ChatCompletion) body := strings.NewReader(tt.body) @@ -2550,7 +2550,7 @@ func bodyMutTransformer(_ *testing.T) cmp.Option { // TestResponseModel_GCPVertexAI tests that GCP Vertex AI returns the request model (no response field) func TestResponseModel_GCPVertexAI(t *testing.T) { modelName := "gemini-1.5-pro-002" - translator := NewChatCompletionOpenAIToGCPVertexAITranslator(modelName) + translator := NewChatCompletionOpenAIToGCPVertexAITranslator(modelName, nil) // Initialize translator with the model req := &openai.ChatCompletionRequest{ diff --git a/internal/translator/tokenize_gcpvertexai.go b/internal/translator/tokenize_gcpvertexai.go index e22dfce61b..7c83f8725b 100644 --- a/internal/translator/tokenize_gcpvertexai.go +++ b/internal/translator/tokenize_gcpvertexai.go @@ -39,7 +39,7 @@ type ToGCPVertexAIV1Tokenize struct { // tokenizeToGeminiCountToken converts an OpenAI tokenize chat request to GCP Gemini CountTokens format. func (o *ToGCPVertexAIV1Tokenize) tokenizeToGeminiCountToken(tokenizeChatReq *tokenize.ChatRequest, requestModel internalapi.RequestModel) (*gcp.CountTokenRequest, error) { // Convert messages to Gemini Contents and SystemInstruction. - contents, systemInstruction, err := openAIMessagesToGeminiContents(tokenizeChatReq.Messages, requestModel) + contents, systemInstruction, err := openAIMessagesToGeminiContents(tokenizeChatReq.Messages, requestModel, nil) if err != nil { return nil, err } @@ -48,7 +48,7 @@ func (o *ToGCPVertexAIV1Tokenize) tokenizeToGeminiCountToken(tokenizeChatReq *to } // Some models support only partialJSONSchema. - parametersJSONSchemaAvailable := responseJSONSchemaAvailable(requestModel) + parametersJSONSchemaAvailable := responseJSONSchemaAvailable(requestModel, nil) // Convert OpenAI tools to Gemini tools. tools, err := openAIToolsToGeminiTools(tokenizeChatReq.Tools, parametersJSONSchemaAvailable) if err != nil { diff --git a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_aiservicebackends.yaml b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_aiservicebackends.yaml index 64a3fa17c1..c8d9e2a5f8 100644 --- a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_aiservicebackends.yaml +++ b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_aiservicebackends.yaml @@ -310,6 +310,61 @@ spec: - name x-kubernetes-list-type: map type: object + modelTranslationHints: + description: |- + ModelTranslationHints declares translation hints for models routed through this backend, + keyed by the exact ModelNameOverride used in an AIGatewayRouteRuleBackendRef that targets + this backend. This is the single source of truth for a model's hints on this backend: any + AIGatewayRoute, in any namespace, that references this backend with a matching + ModelNameOverride picks up these hints without repeating them. + + Since an AIServiceBackend has exactly one Schema, hints declared here are inherently scoped + to that provider — the same logical model hosted on a different AIServiceBackend (e.g. the + same Claude model on AWS Bedrock vs. GCP Vertex AI) is configured as a separate entry on + that other backend, which is correct because providers can genuinely support different + features for what is nominally the same model. + items: + description: |- + ModelTranslationHintsEntry associates ModelTranslationHints with a specific model routed + through this backend. + properties: + maxOutputTokens: + description: |- + MaxOutputTokens is the maximum number of tokens the model can produce in a single response. + When set, translators use this as the default max_tokens value for APIs that require it + (e.g., the Anthropic Messages API) when the client request omits max_tokens. + format: int64 + minimum: 1 + type: integer + modelNameOverride: + description: |- + ModelNameOverride identifies which model these hints apply to. Must match the + ModelNameOverride configured on an AIGatewayRouteRuleBackendRef that targets this backend + for the hints to take effect. + minLength: 1 + type: string + supportsOutputConfig: + description: |- + SupportsOutputConfig indicates whether the model accepts an Anthropic-style output_config + field for structured output (e.g., Claude Opus/Sonnet 4.5 and later). + type: boolean + supportsReasoningEffort: + description: |- + SupportsReasoningEffort indicates whether the model accepts a reasoning effort parameter + (Gemini thinking level / Anthropic output_config.effort). + type: boolean + supportsResponseJsonSchema: + description: |- + SupportsResponseJSONSchema indicates whether the model accepts a JSON Schema object in + response_format (as opposed to only {"type": "json_object"}). + type: boolean + required: + - modelNameOverride + type: object + type: array + x-kubernetes-list-map-keys: + - modelNameOverride + x-kubernetes-list-type: map schema: description: |- APISchema specifies the API schema of the output format of requests from @@ -780,6 +835,61 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + modelTranslationHints: + description: |- + ModelTranslationHints declares translation hints for models routed through this backend, + keyed by the exact ModelNameOverride used in an AIGatewayRouteRuleBackendRef that targets + this backend. This is the single source of truth for a model's hints on this backend: any + AIGatewayRoute, in any namespace, that references this backend with a matching + ModelNameOverride picks up these hints without repeating them. + + Since an AIServiceBackend has exactly one Schema, hints declared here are inherently scoped + to that provider — the same logical model hosted on a different AIServiceBackend (e.g. the + same Claude model on AWS Bedrock vs. GCP Vertex AI) is configured as a separate entry on + that other backend, which is correct because providers can genuinely support different + features for what is nominally the same model. + items: + description: |- + ModelTranslationHintsEntry associates ModelTranslationHints with a specific model routed + through this backend. + properties: + maxOutputTokens: + description: |- + MaxOutputTokens is the maximum number of tokens the model can produce in a single response. + When set, translators use this as the default max_tokens value for APIs that require it + (e.g., the Anthropic Messages API) when the client request omits max_tokens. + format: int64 + minimum: 1 + type: integer + modelNameOverride: + description: |- + ModelNameOverride identifies which model these hints apply to. Must match the + ModelNameOverride configured on an AIGatewayRouteRuleBackendRef that targets this backend + for the hints to take effect. + minLength: 1 + type: string + supportsOutputConfig: + description: |- + SupportsOutputConfig indicates whether the model accepts an Anthropic-style output_config + field for structured output (e.g., Claude Opus/Sonnet 4.5 and later). + type: boolean + supportsReasoningEffort: + description: |- + SupportsReasoningEffort indicates whether the model accepts a reasoning effort parameter + (Gemini thinking level / Anthropic output_config.effort). + type: boolean + supportsResponseJsonSchema: + description: |- + SupportsResponseJSONSchema indicates whether the model accepts a JSON Schema object in + response_format (as opposed to only {"type": "json_object"}). + type: boolean + required: + - modelNameOverride + type: object + type: array + x-kubernetes-list-map-keys: + - modelNameOverride + x-kubernetes-list-type: map schema: description: |- APISchema specifies the API schema of the output format of requests from diff --git a/site/docs/api/api.mdx b/site/docs/api/api.mdx index 03621506b7..87ad6813d7 100644 --- a/site/docs/api/api.mdx +++ b/site/docs/api/api.mdx @@ -611,6 +611,8 @@ QuotaPolicyList contains a list of QuotaPolicy - [MCPRouteSpec](#github-com-envoyproxy-ai-gateway-api-v1alpha1-mcproutespec) - [MCPRouteStatus](#github-com-envoyproxy-ai-gateway-api-v1alpha1-mcproutestatus) - [MCPToolFilter](#github-com-envoyproxy-ai-gateway-api-v1alpha1-mcptoolfilter) +- [ModelTranslationHints](#github-com-envoyproxy-ai-gateway-api-v1alpha1-modeltranslationhints) +- [ModelTranslationHintsEntry](#github-com-envoyproxy-ai-gateway-api-v1alpha1-modeltranslationhintsentry) - [PerModelQuota](#github-com-envoyproxy-ai-gateway-api-v1alpha1-permodelquota) - [ProtectedResourceMetadata](#github-com-envoyproxy-ai-gateway-api-v1alpha1-protectedresourcemetadata) - [QuotaBucketMode](#github-com-envoyproxy-ai-gateway-api-v1alpha1-quotabucketmode) @@ -856,6 +858,11 @@ AIServiceBackendSpec details the AIServiceBackend configuration. type="[HTTPBodyMutation](#github-com-envoyproxy-ai-gateway-api-v1alpha1-httpbodymutation)" required="false" description="BodyMutation defines the mutation of HTTP request body JSON fields that will be applied to the request
before sending it to the backend." +/> @@ -2340,6 +2347,89 @@ a tool must match an include rule AND not match any exclude rule to be allowed. /> +#### ModelTranslationHints + + + +**Appears in:** +- [ModelTranslationHintsEntry](#github-com-envoyproxy-ai-gateway-api-v1alpha1-modeltranslationhintsentry) + +ModelTranslationHints provides model-specific hints to guide request and response translation. +When set, translators use these values instead of model-name heuristics. All fields are +optional; a nil field falls back to the built-in heuristic for backward compatibility. + +These hints are typically populated by a control plane from a model registry or catalog that +tracks per-model metadata such as maximum output token limits and supported features. + +##### Fields + + + + + + +#### ModelTranslationHintsEntry + + + +**Appears in:** +- [AIServiceBackendSpec](#github-com-envoyproxy-ai-gateway-api-v1alpha1-aiservicebackendspec) + +ModelTranslationHintsEntry associates ModelTranslationHints with a specific model routed +through this backend. + +##### Fields + + + + + + #### PerModelQuota @@ -3199,6 +3289,8 @@ MCPRouteList contains a list of MCPRoute. - [MCPRouteSpec](#github-com-envoyproxy-ai-gateway-api-v1beta1-mcproutespec) - [MCPRouteStatus](#github-com-envoyproxy-ai-gateway-api-v1beta1-mcproutestatus) - [MCPToolFilter](#github-com-envoyproxy-ai-gateway-api-v1beta1-mcptoolfilter) +- [ModelTranslationHints](#github-com-envoyproxy-ai-gateway-api-v1beta1-modeltranslationhints) +- [ModelTranslationHintsEntry](#github-com-envoyproxy-ai-gateway-api-v1beta1-modeltranslationhintsentry) - [ProtectedResourceMetadata](#github-com-envoyproxy-ai-gateway-api-v1beta1-protectedresourcemetadata) - [ToolCall](#github-com-envoyproxy-ai-gateway-api-v1beta1-toolcall) - [VersionedAPISchema](#github-com-envoyproxy-ai-gateway-api-v1beta1-versionedapischema) @@ -3441,6 +3533,11 @@ AIServiceBackendSpec details the AIServiceBackend configuration. type="[HTTPHeaderValueFilter](#github-com-envoyproxy-ai-gateway-api-v1beta1-httpheadervaluefilter) array" required="false" description="HeaderValueFilters filter individual values out of multi-valued request headers before sending
the request to this backend, so that one value an upstream provider does not accept does not
fail the whole request. At most one filter per header name.
Only GCPAnthropic and AWSAnthropic backends honor this, and today only for the
`anthropic-beta` header — a filter naming any other header is accepted but has no effect.
Setting it on a backend with any other schema is rejected." +/> @@ -5208,6 +5305,89 @@ a tool must match an include rule AND not match any exclude rule to be allowed. /> +#### ModelTranslationHints + + + +**Appears in:** +- [ModelTranslationHintsEntry](#github-com-envoyproxy-ai-gateway-api-v1beta1-modeltranslationhintsentry) + +ModelTranslationHints provides model-specific hints to guide request and response translation. +When set, translators use these values instead of model-name heuristics. All fields are +optional; a nil field falls back to the built-in heuristic for backward compatibility. + +These hints are typically populated by a control plane from a model registry or catalog that +tracks per-model metadata such as maximum output token limits and supported features. + +##### Fields + + + + + + +#### ModelTranslationHintsEntry + + + +**Appears in:** +- [AIServiceBackendSpec](#github-com-envoyproxy-ai-gateway-api-v1beta1-aiservicebackendspec) + +ModelTranslationHintsEntry associates ModelTranslationHints with a specific model routed +through this backend. + +##### Fields + + + + + + #### ProtectedResourceMetadata