From 9cb6818fdad02fec642e12737a701e48471a0d3b Mon Sep 17 00:00:00 2001 From: yxia216 Date: Tue, 14 Jul 2026 12:26:53 -0400 Subject: [PATCH 1/3] initial-implementation-for-cap-ware Signed-off-by: yxia216 --- api/v1alpha1/ai_service_backend.go | 66 +++++++++++ api/v1alpha1/zz_generated.deepcopy.go | 58 ++++++++++ api/v1beta1/ai_service_backend.go | 66 +++++++++++ api/v1beta1/zz_generated.deepcopy.go | 58 ++++++++++ internal/controller/gateway.go | 25 ++++ internal/controller/gateway_test.go | 75 ++++++++++++ internal/endpointspec/endpointspec.go | 70 ++++++----- internal/endpointspec/endpointspec_test.go | 46 ++++---- internal/extproc/processor_impl.go | 2 +- internal/filterapi/filterconfig.go | 28 +++++ internal/translator/anthropic_helper.go | 27 ++++- internal/translator/anthropic_helper_test.go | 65 +++++++++-- internal/translator/gemini_helper.go | 42 ++++--- internal/translator/gemini_helper_test.go | 32 ++++- internal/translator/openai_awsanthropic.go | 6 +- .../translator/openai_awsanthropic_test.go | 34 +++--- internal/translator/openai_gcpanthropic.go | 6 +- .../translator/openai_gcpanthropic_test.go | 70 +++++------ internal/translator/openai_gcpvertexai.go | 14 ++- .../openai_gcpvertexai_embeddings.go | 3 +- .../translator/openai_gcpvertexai_test.go | 24 ++-- internal/translator/tokenize_gcpvertexai.go | 4 +- ...teway.envoyproxy.io_aiservicebackends.yaml | 109 ++++++++++++++++++ site/docs/api/api.mdx | 96 +++++++++++++++ 24 files changed, 865 insertions(+), 161 deletions(-) diff --git a/api/v1alpha1/ai_service_backend.go b/api/v1alpha1/ai_service_backend.go index cddcc253f9..20c15a2c4a 100644 --- a/api/v1alpha1/ai_service_backend.go +++ b/api/v1alpha1/ai_service_backend.go @@ -76,10 +76,76 @@ type AIServiceBackendSpec struct { // +optional BodyMutation *HTTPBodyMutation `json:"bodyMutation,omitempty"` + // ModelTranslationHints declares translation hints for models routed through this backend, + // keyed by the exact ModelNameOverride used in an AIGatewayRouteRuleBackendRef that targets + // this backend. This is the single source of truth for a model's hints on this backend: any + // AIGatewayRoute, in any namespace, that references this backend with a matching + // ModelNameOverride picks up these hints without repeating them. + // + // Since an AIServiceBackend has exactly one Schema, hints declared here are inherently scoped + // to that provider — the same logical model hosted on a different AIServiceBackend (e.g. the + // same Claude model on AWS Bedrock vs. GCP Vertex AI) is configured as a separate entry on + // that other backend, which is correct because providers can genuinely support different + // features for what is nominally the same model. + // + // +optional + // +listType=map + // +listMapKey=modelNameOverride + ModelTranslationHints []ModelTranslationHintsEntry `json:"modelTranslationHints,omitempty"` + // TODO: maybe add backend-level LLMRequestCost configuration that overrides the AIGatewayRoute-level LLMRequestCost. // That may be useful for the backend that has a different cost calculation logic. } +// ModelTranslationHintsEntry associates ModelTranslationHints with a specific model routed +// through this backend. +type ModelTranslationHintsEntry struct { + // ModelNameOverride identifies which model these hints apply to. Must match the + // ModelNameOverride configured on an AIGatewayRouteRuleBackendRef that targets this backend + // for the hints to take effect. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + ModelNameOverride string `json:"modelNameOverride"` + + // ModelTranslationHints are the hints that apply to this model on this backend. + ModelTranslationHints `json:",inline"` +} + +// ModelTranslationHints provides model-specific hints to guide request and response translation. +// When set, translators use these values instead of model-name heuristics. All fields are +// optional; a nil field falls back to the built-in heuristic for backward compatibility. +// +// These hints are typically populated by a control plane from a model registry or catalog that +// tracks per-model metadata such as maximum output token limits and supported features. +type ModelTranslationHints struct { + // MaxOutputTokens is the maximum number of tokens the model can produce in a single response. + // When set, translators use this as the default max_tokens value for APIs that require it + // (e.g., the Anthropic Messages API) when the client request omits max_tokens. + // + // +optional + // +kubebuilder:validation:Minimum=1 + MaxOutputTokens *int64 `json:"maxOutputTokens,omitempty"` + + // SupportsResponseJSONSchema indicates whether the model accepts a JSON Schema object in + // response_format (as opposed to only {"type": "json_object"}). + // + // +optional + SupportsResponseJSONSchema *bool `json:"supportsResponseJsonSchema,omitempty"` + + // SupportsReasoningEffort indicates whether the model accepts a reasoning effort parameter + // (Gemini thinking level / Anthropic output_config.effort). + // + // +optional + SupportsReasoningEffort *bool `json:"supportsReasoningEffort,omitempty"` + + // SupportsOutputConfig indicates whether the model accepts an Anthropic-style output_config + // field for structured output (e.g., Claude Opus/Sonnet 4.5 and later). + // + // +optional + SupportsOutputConfig *bool `json:"supportsOutputConfig,omitempty"` +} + // HTTPHeaderMutation defines the mutation of HTTP headers that will be applied to the request type HTTPHeaderMutation struct { // Set overwrites/adds the request with the given header (name, value) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 3c44727050..0b09e7c599 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -338,6 +338,13 @@ func (in *AIServiceBackendSpec) DeepCopyInto(out *AIServiceBackendSpec) { *out = new(HTTPBodyMutation) (*in).DeepCopyInto(*out) } + if in.ModelTranslationHints != nil { + in, out := &in.ModelTranslationHints, &out.ModelTranslationHints + *out = make([]ModelTranslationHintsEntry, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AIServiceBackendSpec. @@ -1610,6 +1617,57 @@ func (in *MCPToolFilter) DeepCopy() *MCPToolFilter { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelTranslationHints) DeepCopyInto(out *ModelTranslationHints) { + *out = *in + if in.MaxOutputTokens != nil { + in, out := &in.MaxOutputTokens, &out.MaxOutputTokens + *out = new(int64) + **out = **in + } + if in.SupportsResponseJSONSchema != nil { + in, out := &in.SupportsResponseJSONSchema, &out.SupportsResponseJSONSchema + *out = new(bool) + **out = **in + } + if in.SupportsReasoningEffort != nil { + in, out := &in.SupportsReasoningEffort, &out.SupportsReasoningEffort + *out = new(bool) + **out = **in + } + if in.SupportsOutputConfig != nil { + in, out := &in.SupportsOutputConfig, &out.SupportsOutputConfig + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelTranslationHints. +func (in *ModelTranslationHints) DeepCopy() *ModelTranslationHints { + if in == nil { + return nil + } + out := new(ModelTranslationHints) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelTranslationHintsEntry) DeepCopyInto(out *ModelTranslationHintsEntry) { + *out = *in + in.ModelTranslationHints.DeepCopyInto(&out.ModelTranslationHints) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelTranslationHintsEntry. +func (in *ModelTranslationHintsEntry) DeepCopy() *ModelTranslationHintsEntry { + if in == nil { + return nil + } + out := new(ModelTranslationHintsEntry) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PerModelQuota) DeepCopyInto(out *PerModelQuota) { *out = *in diff --git a/api/v1beta1/ai_service_backend.go b/api/v1beta1/ai_service_backend.go index ed0d3b8573..80c1be1d4a 100644 --- a/api/v1beta1/ai_service_backend.go +++ b/api/v1beta1/ai_service_backend.go @@ -91,6 +91,72 @@ type AIServiceBackendSpec struct { // +kubebuilder:validation:MaxItems=16 HeaderValueFilters []HTTPHeaderValueFilter `json:"headerValueFilters,omitempty"` + // ModelTranslationHints declares translation hints for models routed through this backend, + // keyed by the exact ModelNameOverride used in an AIGatewayRouteRuleBackendRef that targets + // this backend. This is the single source of truth for a model's hints on this backend: any + // AIGatewayRoute, in any namespace, that references this backend with a matching + // ModelNameOverride picks up these hints without repeating them. + // + // Since an AIServiceBackend has exactly one Schema, hints declared here are inherently scoped + // to that provider — the same logical model hosted on a different AIServiceBackend (e.g. the + // same Claude model on AWS Bedrock vs. GCP Vertex AI) is configured as a separate entry on + // that other backend, which is correct because providers can genuinely support different + // features for what is nominally the same model. + // + // +optional + // +listType=map + // +listMapKey=modelNameOverride + ModelTranslationHints []ModelTranslationHintsEntry `json:"modelTranslationHints,omitempty"` + // TODO: maybe add backend-level LLMRequestCost configuration that overrides the AIGatewayRoute-level LLMRequestCost. // That may be useful for the backend that has a different cost calculation logic. } + +// ModelTranslationHintsEntry associates ModelTranslationHints with a specific model routed +// through this backend. +type ModelTranslationHintsEntry struct { + // ModelNameOverride identifies which model these hints apply to. Must match the + // ModelNameOverride configured on an AIGatewayRouteRuleBackendRef that targets this backend + // for the hints to take effect. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + ModelNameOverride string `json:"modelNameOverride"` + + // ModelTranslationHints are the hints that apply to this model on this backend. + ModelTranslationHints `json:",inline"` +} + +// ModelTranslationHints provides model-specific hints to guide request and response translation. +// When set, translators use these values instead of model-name heuristics. All fields are +// optional; a nil field falls back to the built-in heuristic for backward compatibility. +// +// These hints are typically populated by a control plane from a model registry or catalog that +// tracks per-model metadata such as maximum output token limits and supported features. +type ModelTranslationHints struct { + // MaxOutputTokens is the maximum number of tokens the model can produce in a single response. + // When set, translators use this as the default max_tokens value for APIs that require it + // (e.g., the Anthropic Messages API) when the client request omits max_tokens. + // + // +optional + // +kubebuilder:validation:Minimum=1 + MaxOutputTokens *int64 `json:"maxOutputTokens,omitempty"` + + // SupportsResponseJSONSchema indicates whether the model accepts a JSON Schema object in + // response_format (as opposed to only {"type": "json_object"}). + // + // +optional + SupportsResponseJSONSchema *bool `json:"supportsResponseJsonSchema,omitempty"` + + // SupportsReasoningEffort indicates whether the model accepts a reasoning effort parameter + // (Gemini thinking level / Anthropic output_config.effort). + // + // +optional + SupportsReasoningEffort *bool `json:"supportsReasoningEffort,omitempty"` + + // SupportsOutputConfig indicates whether the model accepts an Anthropic-style output_config + // field for structured output (e.g., Claude Opus/Sonnet 4.5 and later). + // + // +optional + SupportsOutputConfig *bool `json:"supportsOutputConfig,omitempty"` +} diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 10823978cf..2fdb0cbaf5 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -345,6 +345,13 @@ func (in *AIServiceBackendSpec) DeepCopyInto(out *AIServiceBackendSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.ModelTranslationHints != nil { + in, out := &in.ModelTranslationHints, &out.ModelTranslationHints + *out = make([]ModelTranslationHintsEntry, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AIServiceBackendSpec. @@ -1767,6 +1774,57 @@ func (in *MCPToolFilter) DeepCopy() *MCPToolFilter { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelTranslationHints) DeepCopyInto(out *ModelTranslationHints) { + *out = *in + if in.MaxOutputTokens != nil { + in, out := &in.MaxOutputTokens, &out.MaxOutputTokens + *out = new(int64) + **out = **in + } + if in.SupportsResponseJSONSchema != nil { + in, out := &in.SupportsResponseJSONSchema, &out.SupportsResponseJSONSchema + *out = new(bool) + **out = **in + } + if in.SupportsReasoningEffort != nil { + in, out := &in.SupportsReasoningEffort, &out.SupportsReasoningEffort + *out = new(bool) + **out = **in + } + if in.SupportsOutputConfig != nil { + in, out := &in.SupportsOutputConfig, &out.SupportsOutputConfig + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelTranslationHints. +func (in *ModelTranslationHints) DeepCopy() *ModelTranslationHints { + if in == nil { + return nil + } + out := new(ModelTranslationHints) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelTranslationHintsEntry) DeepCopyInto(out *ModelTranslationHintsEntry) { + *out = *in + in.ModelTranslationHints.DeepCopyInto(&out.ModelTranslationHints) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelTranslationHintsEntry. +func (in *ModelTranslationHintsEntry) DeepCopy() *ModelTranslationHintsEntry { + if in == nil { + return nil + } + out := new(ModelTranslationHintsEntry) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProtectedResourceMetadata) DeepCopyInto(out *ProtectedResourceMetadata) { *out = *in diff --git a/internal/controller/gateway.go b/internal/controller/gateway.go index 255d6abc69..e9ed02403a 100644 --- a/internal/controller/gateway.go +++ b/internal/controller/gateway.go @@ -235,6 +235,29 @@ func bodyMutationToFilterAPI(m *aigv1b1.HTTPBodyMutation) *filterapi.HTTPBodyMut return ret } +// modelTranslationHintsForModel finds the ModelTranslationHintsEntry in entries whose +// ModelNameOverride exactly matches modelNameOverride and converts it to filterapi.ModelTranslationHints. +// Returns nil if modelNameOverride is empty or no entry matches, in which case translators fall +// back to their built-in heuristics. +func modelTranslationHintsForModel(entries []aigv1b1.ModelTranslationHintsEntry, modelNameOverride string) *filterapi.ModelTranslationHints { + if modelNameOverride == "" { + return nil + } + for _, entry := range entries { + if entry.ModelNameOverride != modelNameOverride { + continue + } + h := entry.ModelTranslationHints + return &filterapi.ModelTranslationHints{ + MaxOutputTokens: h.MaxOutputTokens, + SupportsResponseJSONSchema: h.SupportsResponseJSONSchema, + SupportsReasoningEffort: h.SupportsReasoningEffort, + SupportsOutputConfig: h.SupportsOutputConfig, + } + } + return nil +} + // validateCELExpression validates and returns a CEL expression for cost calculation. func validateCELExpression(cost aigv1b1.LLMRequestCost) (string, error) { if cost.CEL == nil { @@ -521,6 +544,8 @@ func (c *GatewayController) reconcileFilterConfigSecret( mergedBodyMutation := mergeBodyMutations(routeBodyMutation, backendBodyMutation) b.BodyMutation = bodyMutationToFilterAPI(mergedBodyMutation) + b.TranslationHints = modelTranslationHintsForModel(backendObj.Spec.ModelTranslationHints, backendRef.ModelNameOverride) + b.Schema = schemaToFilterAPI(backendObj.Spec.APISchema) b.HeaderValueFilters = headerValueFiltersToFilterAPI(backendObj.Spec.HeaderValueFilters) } diff --git a/internal/controller/gateway_test.go b/internal/controller/gateway_test.go index ebe381462c..8821256ed6 100644 --- a/internal/controller/gateway_test.go +++ b/internal/controller/gateway_test.go @@ -3699,6 +3699,81 @@ func Test_bodyMutationToFilterAPI(t *testing.T) { } } +func Test_modelTranslationHintsForModel(t *testing.T) { + opusHints := aigv1b1.ModelTranslationHints{ + MaxOutputTokens: ptr.To(int64(32000)), + SupportsResponseJSONSchema: ptr.To(true), + SupportsReasoningEffort: ptr.To(true), + SupportsOutputConfig: ptr.To(false), + } + geminiHints := aigv1b1.ModelTranslationHints{ + MaxOutputTokens: ptr.To(int64(65536)), + } + entries := []aigv1b1.ModelTranslationHintsEntry{ + {ModelNameOverride: "claude-opus-4-6@20250514", ModelTranslationHints: opusHints}, + {ModelNameOverride: "gemini-3-pro", ModelTranslationHints: geminiHints}, + } + + tests := []struct { + name string + entries []aigv1b1.ModelTranslationHintsEntry + modelNameOverride string + expected *filterapi.ModelTranslationHints + }{ + { + name: "empty modelNameOverride", + entries: entries, + modelNameOverride: "", + expected: nil, + }, + { + name: "no entries", + entries: nil, + modelNameOverride: "claude-opus-4-6@20250514", + expected: nil, + }, + { + name: "no matching entry", + entries: entries, + modelNameOverride: "claude-sonnet-4-6@20250514", + expected: nil, + }, + { + name: "matches the first entry among multiple", + entries: entries, + modelNameOverride: "claude-opus-4-6@20250514", + expected: &filterapi.ModelTranslationHints{ + MaxOutputTokens: ptr.To(int64(32000)), + SupportsResponseJSONSchema: ptr.To(true), + SupportsReasoningEffort: ptr.To(true), + SupportsOutputConfig: ptr.To(false), + }, + }, + { + name: "matches a later entry among multiple", + entries: entries, + modelNameOverride: "gemini-3-pro", + expected: &filterapi.ModelTranslationHints{ + MaxOutputTokens: ptr.To(int64(65536)), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := modelTranslationHintsForModel(tt.entries, tt.modelNameOverride) + if tt.expected == nil { + require.Nil(t, result) + return + } + require.NotNil(t, result) + if d := cmp.Diff(tt.expected, result); d != "" { + t.Errorf("modelTranslationHintsForModel() mismatch (-expected +got):\n%s", d) + } + }) + } +} + // TestGatewayController_reconcileFilterConfigSecret_GlobalDefaults tests that // global LLM request costs from GatewayConfig are properly included in the filter config // when no routes override them. diff --git a/internal/endpointspec/endpointspec.go b/internal/endpointspec/endpointspec.go index 545f7c323d..64dffcda8f 100644 --- a/internal/endpointspec/endpointspec.go +++ b/internal/endpointspec/endpointspec.go @@ -60,17 +60,19 @@ type ( // * mutatedBody: The possibly mutated request body as a byte slice. Or nil if no mutation is needed. // * err: An error if parsing fails. ParseBody(body []byte, costConfigured bool) (originalModel internalapi.OriginalModel, req *ReqT, stream bool, mutatedBody []byte, err error) - // GetTranslator selects the appropriate translator based on the output API schema - // and an optional model name override. + // GetTranslator selects the appropriate translator for the given backend. + // + // The full backend is passed (rather than just the schema and model name override) so that + // translators can access model-specific translation hints alongside the schema and model name. // // Parameters: - // * out: The output API schema for which the translator is needed. - // * modelNameOverride: An optional model name to override the one specified in the request. + // * backend: The backend for which the translator is needed. Its Schema selects the translator, + // while ModelNameOverride and TranslationHints tune its behavior. // // Returns: // * translator: The selected translator of type Translator[ReqT, RespT, RespChunkT]. // * err: An error if translator selection fails. - GetTranslator(schema filterapi.VersionedAPISchema, modelNameOverride string) (translator.Translator[ReqT, tracingapi.Span[RespT, RespChunkT]], error) + GetTranslator(backend *filterapi.Backend) (translator.Translator[ReqT, tracingapi.Span[RespT, RespChunkT]], error) // RedactSensitiveInfoFromRequest creates a redacted copy of the request for safe debug logging. // Sensitive content (messages, images, audio, tool parameters, etc.) is replaced with placeholders // containing length and hash information to aid in debugging cache hits/misses and correlation. @@ -164,7 +166,9 @@ func (ChatCompletionsEndpointSpec) ParseMultipartBody([]byte, string, bool) (int } // GetTranslator implements [EndpointSpec.GetTranslator]. -func (ChatCompletionsEndpointSpec) GetTranslator(schema filterapi.VersionedAPISchema, modelNameOverride string) (translator.OpenAIChatCompletionTranslator, error) { +func (ChatCompletionsEndpointSpec) GetTranslator(backend *filterapi.Backend) (translator.OpenAIChatCompletionTranslator, error) { + schema := backend.Schema + modelNameOverride := backend.ModelNameOverride switch schema.Name { case filterapi.APISchemaOpenAI: return translator.NewChatCompletionOpenAIToOpenAITranslator(schema.OpenAIPrefix(), modelNameOverride), nil @@ -173,13 +177,13 @@ func (ChatCompletionsEndpointSpec) GetTranslator(schema filterapi.VersionedAPISc case filterapi.APISchemaAWSBedrock: return translator.NewChatCompletionOpenAIToAWSBedrockTranslator(modelNameOverride), nil case filterapi.APISchemaAWSAnthropic: - return translator.NewChatCompletionOpenAIToAWSAnthropicTranslator(schema.Version, modelNameOverride), nil + return translator.NewChatCompletionOpenAIToAWSAnthropicTranslator(schema.Version, modelNameOverride, backend.TranslationHints), nil case filterapi.APISchemaAzureOpenAI: return translator.NewChatCompletionOpenAIToAzureOpenAITranslator(schema.Version, modelNameOverride), nil case filterapi.APISchemaGCPVertexAI: - return translator.NewChatCompletionOpenAIToGCPVertexAITranslator(modelNameOverride), nil + return translator.NewChatCompletionOpenAIToGCPVertexAITranslator(modelNameOverride, backend.TranslationHints), nil case filterapi.APISchemaGCPAnthropic: - return translator.NewChatCompletionOpenAIToGCPAnthropicTranslator(schema.Version, modelNameOverride), nil + return translator.NewChatCompletionOpenAIToGCPAnthropicTranslator(schema.Version, modelNameOverride, backend.TranslationHints), nil default: return nil, fmt.Errorf("unsupported API schema: backend=%s", schema) } @@ -228,10 +232,11 @@ func (CompletionsEndpointSpec) ParseMultipartBody([]byte, string, bool) (interna } // GetTranslator implements [EndpointSpec.GetTranslator]. -func (CompletionsEndpointSpec) GetTranslator(schema filterapi.VersionedAPISchema, modelNameOverride string) (translator.OpenAICompletionTranslator, error) { +func (CompletionsEndpointSpec) GetTranslator(backend *filterapi.Backend) (translator.OpenAICompletionTranslator, error) { + schema := backend.Schema switch schema.Name { case filterapi.APISchemaOpenAI: - return translator.NewCompletionOpenAIToOpenAITranslator(schema.OpenAIPrefix(), modelNameOverride), nil + return translator.NewCompletionOpenAIToOpenAITranslator(schema.OpenAIPrefix(), backend.ModelNameOverride), nil default: return nil, fmt.Errorf("unsupported API schema: backend=%s", schema) } @@ -265,7 +270,9 @@ func (EmbeddingsEndpointSpec) ParseMultipartBody([]byte, string, bool) (internal } // GetTranslator implements [EndpointSpec.GetTranslator]. -func (EmbeddingsEndpointSpec) GetTranslator(schema filterapi.VersionedAPISchema, modelNameOverride string) (translator.OpenAIEmbeddingTranslator, error) { +func (EmbeddingsEndpointSpec) GetTranslator(backend *filterapi.Backend) (translator.OpenAIEmbeddingTranslator, error) { + schema := backend.Schema + modelNameOverride := backend.ModelNameOverride switch schema.Name { case filterapi.APISchemaOpenAI: return translator.NewEmbeddingOpenAIToOpenAITranslator(schema.OpenAIPrefix(), modelNameOverride), nil @@ -320,10 +327,11 @@ func (ImageGenerationEndpointSpec) ParseMultipartBody([]byte, string, bool) (int } // GetTranslator implements [EndpointSpec.GetTranslator]. -func (ImageGenerationEndpointSpec) GetTranslator(schema filterapi.VersionedAPISchema, modelNameOverride string) (translator.OpenAIImageGenerationTranslator, error) { +func (ImageGenerationEndpointSpec) GetTranslator(backend *filterapi.Backend) (translator.OpenAIImageGenerationTranslator, error) { + schema := backend.Schema switch schema.Name { case filterapi.APISchemaOpenAI: - return translator.NewImageGenerationOpenAIToOpenAITranslator(schema.OpenAIPrefix(), modelNameOverride), nil + return translator.NewImageGenerationOpenAIToOpenAITranslator(schema.OpenAIPrefix(), backend.ModelNameOverride), nil default: return nil, fmt.Errorf("unsupported API schema: backend=%s", schema) } @@ -355,14 +363,15 @@ func (ResponsesEndpointSpec) ParseMultipartBody([]byte, string, bool) (internala } // GetTranslator implements [EndpointSpec.GetTranslator]. -func (ResponsesEndpointSpec) GetTranslator(schema filterapi.VersionedAPISchema, modelNameOverride string) (translator.OpenAIResponsesTranslator, error) { +func (ResponsesEndpointSpec) GetTranslator(backend *filterapi.Backend) (translator.OpenAIResponsesTranslator, error) { + schema := backend.Schema switch schema.Name { case filterapi.APISchemaOpenAI: - return translator.NewResponsesOpenAIToOpenAITranslator(schema.OpenAIPrefix(), modelNameOverride), nil + return translator.NewResponsesOpenAIToOpenAITranslator(schema.OpenAIPrefix(), backend.ModelNameOverride), nil case filterapi.APISchemaAWSOpenAI: - return translator.NewResponsesOpenAIToAWSOpenAITranslator(schema.OpenAIPrefix(), modelNameOverride), nil + return translator.NewResponsesOpenAIToAWSOpenAITranslator(schema.OpenAIPrefix(), backend.ModelNameOverride), nil case filterapi.APISchemaAzureOpenAI: - return translator.NewResponsesOpenAIToAzureOpenAITranslator(schema.Version, modelNameOverride), nil + return translator.NewResponsesOpenAIToAzureOpenAITranslator(schema.Version, backend.ModelNameOverride), nil default: return nil, fmt.Errorf("unsupported API schema: backend=%s", schema) } @@ -406,7 +415,9 @@ func (MessagesEndpointSpec) ParseMultipartBody([]byte, string, bool) (internalap } // GetTranslator implements [EndpointSpec.GetTranslator]. -func (MessagesEndpointSpec) GetTranslator(schema filterapi.VersionedAPISchema, modelNameOverride string) (translator.AnthropicMessagesTranslator, error) { +func (MessagesEndpointSpec) GetTranslator(backend *filterapi.Backend) (translator.AnthropicMessagesTranslator, error) { + schema := backend.Schema + modelNameOverride := backend.ModelNameOverride // Messages processor only supports Anthropic-native translators. switch schema.Name { case filterapi.APISchemaGCPAnthropic: @@ -501,10 +512,11 @@ func (RerankEndpointSpec) ParseMultipartBody([]byte, string, bool) (internalapi. } // GetTranslator implements [EndpointSpec.GetTranslator]. -func (RerankEndpointSpec) GetTranslator(schema filterapi.VersionedAPISchema, modelNameOverride string) (translator.CohereRerankTranslator, error) { +func (RerankEndpointSpec) GetTranslator(backend *filterapi.Backend) (translator.CohereRerankTranslator, error) { + schema := backend.Schema switch schema.Name { case filterapi.APISchemaCohere: - return translator.NewRerankCohereToCohereTranslator(schema.Version, modelNameOverride), nil + return translator.NewRerankCohereToCohereTranslator(schema.Version, backend.ModelNameOverride), nil default: return nil, fmt.Errorf("unsupported API schema: backend=%s", schema) } @@ -900,14 +912,14 @@ func (SpeechEndpointSpec) ParseMultipartBody([]byte, string, bool) (internalapi. // GetTranslator implements [EndpointSpec.GetTranslator]. func (SpeechEndpointSpec) GetTranslator( - schema filterapi.VersionedAPISchema, - modelNameOverride string, + backend *filterapi.Backend, ) (translator.OpenAISpeechTranslator, error) { + schema := backend.Schema switch schema.Name { case filterapi.APISchemaOpenAI: return translator.NewSpeechOpenAIToOpenAITranslator( schema.OpenAIPrefix(), - modelNameOverride, + backend.ModelNameOverride, ), nil default: return nil, fmt.Errorf("unsupported API schema for speech: backend=%s", schema) @@ -1035,11 +1047,12 @@ func (TranscriptionEndpointSpec) ParseMultipartBody( // GetTranslator implements [Spec.GetTranslator]. func (TranscriptionEndpointSpec) GetTranslator( - schema filterapi.VersionedAPISchema, modelNameOverride string, + backend *filterapi.Backend, ) (translator.OpenAIAudioTranscriptionTranslator, error) { + schema := backend.Schema switch schema.Name { case filterapi.APISchemaOpenAI: - return translator.NewTranscriptionOpenAIToOpenAITranslator(schema.OpenAIPrefix(), modelNameOverride), nil + return translator.NewTranscriptionOpenAIToOpenAITranslator(schema.OpenAIPrefix(), backend.ModelNameOverride), nil default: return nil, fmt.Errorf("unsupported API schema for audio transcription: backend=%s", schema) } @@ -1140,11 +1153,12 @@ func (TranslationEndpointSpec) ParseMultipartBody( // GetTranslator implements [Spec.GetTranslator]. func (TranslationEndpointSpec) GetTranslator( - schema filterapi.VersionedAPISchema, modelNameOverride string, + backend *filterapi.Backend, ) (translator.OpenAIAudioTranslationTranslator, error) { + schema := backend.Schema switch schema.Name { case filterapi.APISchemaOpenAI: - return translator.NewTranslationOpenAIToOpenAITranslator(schema.OpenAIPrefix(), modelNameOverride), nil + return translator.NewTranslationOpenAIToOpenAITranslator(schema.OpenAIPrefix(), backend.ModelNameOverride), nil default: return nil, fmt.Errorf("unsupported API schema for audio translation: backend=%s", schema) } diff --git a/internal/endpointspec/endpointspec_test.go b/internal/endpointspec/endpointspec_test.go index 46a9158273..a7f8aa29e0 100644 --- a/internal/endpointspec/endpointspec_test.go +++ b/internal/endpointspec/endpointspec_test.go @@ -97,19 +97,19 @@ func TestChatCompletionsEndpointSpec_GetTranslator(t *testing.T) { s := schema t.Run("supported_"+string(s.Name), func(t *testing.T) { t.Parallel() - translator, err := spec.GetTranslator(s, "override") + translator, err := spec.GetTranslator(&filterapi.Backend{Schema: s, ModelNameOverride: "override"}) require.NoError(t, err) require.NotNil(t, translator) }) } t.Run("unsupported", func(t *testing.T) { - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: "Unknown"}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: "Unknown"}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "unsupported API schema") }) t.Run("AWSOpenAI request", func(t *testing.T) { - awsTranslator, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaAWSOpenAI}, "") + awsTranslator, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAWSOpenAI}}) require.NoError(t, err) original := []byte(`{"model":"us.openai.gpt-5.6-luna","messages":[]}`) headers, body, err := awsTranslator.RequestBody(original, &openai.ChatCompletionRequest{Model: "us.openai.gpt-5.6-luna"}, false) @@ -144,10 +144,10 @@ func TestCompletionsEndpointSpec_ParseBody(t *testing.T) { func TestCompletionsEndpointSpec_GetTranslator(t *testing.T) { spec := CompletionsEndpointSpec{} - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, ModelNameOverride: "override"}) require.NoError(t, err) - _, err = spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaAWSBedrock}, "override") + _, err = spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAWSBedrock}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "unsupported API schema") } @@ -203,14 +203,14 @@ func TestEmbeddingsEndpointSpec_GetTranslator(t *testing.T) { s := schema t.Run("supported_"+string(s.Name), func(t *testing.T) { t.Parallel() - tr, err := spec.GetTranslator(s, "override") + tr, err := spec.GetTranslator(&filterapi.Backend{Schema: s, ModelNameOverride: "override"}) require.NoError(t, err) require.NotNil(t, tr) }) } t.Run("unsupported", func(t *testing.T) { - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaCohere}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaCohere}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "unsupported API schema") }) } @@ -239,10 +239,10 @@ func TestImageGenerationEndpointSpec_ParseBody(t *testing.T) { func TestImageGenerationEndpointSpec_GetTranslator(t *testing.T) { spec := ImageGenerationEndpointSpec{} - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, ModelNameOverride: "override"}) require.NoError(t, err) - _, err = spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaAzureOpenAI}, "override") + _, err = spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAzureOpenAI}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "unsupported API schema") } @@ -284,12 +284,12 @@ func TestMessagesEndpointSpec_GetTranslator(t *testing.T) { {Name: filterapi.APISchemaOpenAI}, // This is for OpenAI-schema backends like vLLM that support the /v1/messages endpoint {Name: filterapi.APISchemaAWSBedrock}, // This is for AWS Bedrock translation from the /v1/messages endpoint } { - translator, err := spec.GetTranslator(schema, "override") + translator, err := spec.GetTranslator(&filterapi.Backend{Schema: schema, ModelNameOverride: "override"}) require.NoError(t, err) require.NotNil(t, translator) } - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaCohere}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaCohere}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "only supports") } @@ -317,10 +317,10 @@ func TestRerankEndpointSpec_ParseBody(t *testing.T) { func TestRerankEndpointSpec_GetTranslator(t *testing.T) { spec := RerankEndpointSpec{} - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaCohere}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaCohere}, ModelNameOverride: "override"}) require.NoError(t, err) - _, err = spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, "override") + _, err = spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "unsupported API schema") } @@ -426,13 +426,13 @@ func TestResponsesEndpointSpec_ParseBody(t *testing.T) { func TestResponsesEndpointSpec_GetTranslator(t *testing.T) { spec := ResponsesEndpointSpec{} - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, ModelNameOverride: "override"}) require.NoError(t, err) - _, err = spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaAzureOpenAI}, "override") + _, err = spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAzureOpenAI}, ModelNameOverride: "override"}) require.NoError(t, err) - awsTranslator, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaAWSOpenAI}, "us.openai.gpt-5.6-luna") + awsTranslator, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAWSOpenAI}, ModelNameOverride: "us.openai.gpt-5.6-luna"}) require.NoError(t, err) headers, body, err := awsTranslator.RequestBody( []byte(`{"model":"gpt-5.6-luna","input":"hello"}`), @@ -443,7 +443,7 @@ func TestResponsesEndpointSpec_GetTranslator(t *testing.T) { require.Equal(t, internalapi.Header{":path", "/openai/v1/responses"}, headers[0]) require.JSONEq(t, `{"model":"us.openai.gpt-5.6-luna","input":"hello"}`, string(body)) - awsTranslator, err = spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaAWSOpenAI}, "") + awsTranslator, err = spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAWSOpenAI}}) require.NoError(t, err) original := []byte(`{"model":"us.openai.gpt-5.6-luna","input":"hello"}`) _, body, err = awsTranslator.RequestBody(original, &openai.ResponseRequest{Model: "us.openai.gpt-5.6-luna"}, false) @@ -1135,10 +1135,10 @@ func TestSpeechEndpointSpec_ParseBody(t *testing.T) { func TestSpeechEndpointSpec_GetTranslator(t *testing.T) { spec := SpeechEndpointSpec{} - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, ModelNameOverride: "override"}) require.NoError(t, err) - _, err = spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaAzureOpenAI}, "override") + _, err = spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAzureOpenAI}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "unsupported API schema for speech") } @@ -1395,10 +1395,10 @@ func TestTranscriptionEndpointSpec_ParseMultipartBody(t *testing.T) { func TestTranscriptionEndpointSpec_GetTranslator(t *testing.T) { spec := TranscriptionEndpointSpec{} - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, ModelNameOverride: "override"}) require.NoError(t, err) - _, err = spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaAzureOpenAI}, "override") + _, err = spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAzureOpenAI}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "unsupported API schema for audio transcription") } @@ -1503,10 +1503,10 @@ func TestTranslationEndpointSpec_ParseMultipartBody(t *testing.T) { func TestTranslationEndpointSpec_GetTranslator(t *testing.T) { spec := TranslationEndpointSpec{} - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, ModelNameOverride: "override"}) require.NoError(t, err) - _, err = spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaAzureOpenAI}, "override") + _, err = spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAzureOpenAI}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "unsupported API schema for audio translation") } diff --git a/internal/extproc/processor_impl.go b/internal/extproc/processor_impl.go index 5bcdb2d269..df3ca883f9 100644 --- a/internal/extproc/processor_impl.go +++ b/internal/extproc/processor_impl.go @@ -713,7 +713,7 @@ func (u *upstreamProcessor[ReqT, RespT, RespChunkT, EndpointSpecT]) SetBackend(c } u.parent = rp // Set parent before GetTranslator so it can access rp.eh - u.translator, err = u.parent.eh.GetTranslator(backend.Backend.Schema, u.modelNameOverride) + u.translator, err = u.parent.eh.GetTranslator(backend.Backend) if err != nil { return fmt.Errorf("failed to create translator for backend %s: %w", backend.Backend.Name, err) } diff --git a/internal/filterapi/filterconfig.go b/internal/filterapi/filterconfig.go index f2b7775b6f..32220b079e 100644 --- a/internal/filterapi/filterconfig.go +++ b/internal/filterapi/filterconfig.go @@ -207,6 +207,34 @@ type Backend struct { // HeaderValueFilters filter individual values out of multi-valued request headers before sending // the request to the backend. Optional. HeaderValueFilters []HTTPHeaderValueFilter `json:"headerValueFilters,omitempty"` + + // TranslationHints provides optional model-specific hints to guide request and response + // translation for this model-backend pairing. When nil (or a given field is nil), translators + // fall back to their built-in model-name heuristics. Optional. + TranslationHints *ModelTranslationHints `json:"translationHints,omitempty"` +} + +// ModelTranslationHints provides model-specific hints to guide request and response translation +// for a model-backend pairing. When set, translators use these values instead of model-name +// heuristics. All fields are optional; a nil field falls back to the built-in heuristic for +// backward compatibility. +// +// This mirrors the aigv1b1.ModelTranslationHints CRD type and is serialized into the filter +// config so translators can read it at runtime without Kubernetes access. +type ModelTranslationHints struct { + // MaxOutputTokens is the maximum number of tokens the model can produce in a single response. + // When set, translators use this as the default max_tokens value for APIs that require it + // (e.g., the Anthropic Messages API) when the client request omits max_tokens. + MaxOutputTokens *int64 `json:"maxOutputTokens,omitempty"` + // SupportsResponseJSONSchema indicates whether the model accepts a JSON Schema object in + // response_format (as opposed to only {"type": "json_object"}). + SupportsResponseJSONSchema *bool `json:"supportsResponseJsonSchema,omitempty"` + // SupportsReasoningEffort indicates whether the model accepts a reasoning effort parameter + // (Gemini thinking level / Anthropic output_config.effort). + SupportsReasoningEffort *bool `json:"supportsReasoningEffort,omitempty"` + // SupportsOutputConfig indicates whether the model accepts an Anthropic-style output_config + // field for structured output (e.g., Claude Opus/Sonnet 4.5 and later). + SupportsOutputConfig *bool `json:"supportsOutputConfig,omitempty"` } // BackendAuth corresponds partially to BackendSecurityPolicy in api/v1alpha1/api.go. diff --git a/internal/translator/anthropic_helper.go b/internal/translator/anthropic_helper.go index 3697baa5fb..c126bdccab 100644 --- a/internal/translator/anthropic_helper.go +++ b/internal/translator/anthropic_helper.go @@ -660,7 +660,13 @@ var gcpOutputConfigModels = []string{ "mythos-preview", // Claude Mythos Preview } -func outputConfigAvailable(apiSchema filterapi.APISchemaName, model internalapi.RequestModel) bool { +// outputConfigAvailable reports whether the model supports Anthropic structured output. +// An explicit backend-scoped hint is authoritative; otherwise the provider-specific +// model-name heuristic preserves backward compatibility. +func outputConfigAvailable(apiSchema filterapi.APISchemaName, model internalapi.RequestModel, hints *filterapi.ModelTranslationHints) bool { + if hints != nil && hints.SupportsOutputConfig != nil { + return *hints.SupportsOutputConfig + } switch apiSchema { case filterapi.APISchemaGCPAnthropic: return modelContainsAny(model, gcpOutputConfigModels) @@ -687,7 +693,13 @@ var effortModels = []string{ "mythos-preview", // Claude Mythos Preview } -func effortAvailable(model internalapi.RequestModel) bool { +// effortAvailable reports whether the model supports the output_config.effort parameter. +// When hints declare SupportsReasoningEffort, that value is authoritative; otherwise it falls back +// to the model-name heuristic for backward compatibility. +func effortAvailable(model internalapi.RequestModel, hints *filterapi.ModelTranslationHints) bool { + if hints != nil && hints.SupportsReasoningEffort != nil { + return *hints.SupportsReasoningEffort + } return modelContainsAny(model, effortModels) } @@ -714,13 +726,16 @@ func mapReasoningEffortToOutputConfigEffort(reasonEffort openai.ReasoningEffort) // into the parameter struct required by the Anthropic SDK. // The apiSchema parameter indicates the backend API schema (e.g., APISchemaAWSAnthropic, // APISchemaGCPAnthropic) and is used to gate backend-specific feature support. -func buildAnthropicParams(openAIReq *openai.ChatCompletionRequest, apiSchema filterapi.APISchemaName, modelNameOverride internalapi.ModelNameOverride) (params *anthropic.MessageNewParams, err error) { +func buildAnthropicParams(openAIReq *openai.ChatCompletionRequest, apiSchema filterapi.APISchemaName, modelNameOverride internalapi.ModelNameOverride, hints *filterapi.ModelTranslationHints) (params *anthropic.MessageNewParams, err error) { // 1. Handle simple parameters. // max_tokens is required by the Anthropic API but optional in the OpenAI API. - // If not set, pass 0 and let the Anthropic API reject the request. + // If the client omits it, default to the model's declared max output tokens when hints provide + // it; otherwise pass 0 and let the Anthropic API reject the request (existing behavior). var maxTokensVal int64 if maxTokens := cmp.Or(openAIReq.MaxCompletionTokens, openAIReq.MaxTokens); maxTokens != nil { maxTokensVal = *maxTokens + } else if hints != nil && hints.MaxOutputTokens != nil { + maxTokensVal = *hints.MaxOutputTokens } // Translate openAI contents to anthropic params. @@ -754,7 +769,7 @@ func buildAnthropicParams(openAIReq *openai.ChatCompletionRequest, apiSchema fil if modelNameOverride != "" { featureCheckModel = modelNameOverride } - if openAIReq.ResponseFormat != nil && openAIReq.ResponseFormat.OfJSONSchema != nil && outputConfigAvailable(apiSchema, featureCheckModel) { + if openAIReq.ResponseFormat != nil && openAIReq.ResponseFormat.OfJSONSchema != nil && outputConfigAvailable(apiSchema, featureCheckModel, hints) { // Validate that the OpenAI JSON schema is an object while retaining its // original bytes. Anthropic's SDK sorts map keys when marshaling, which // would otherwise change the property order seen by Claude. @@ -776,7 +791,7 @@ func buildAnthropicParams(openAIReq *openai.ChatCompletionRequest, apiSchema fil } // Map OpenAI reasoning_effort to Anthropic output_config.effort. - if openAIReq.ReasoningEffort != "" && effortAvailable(featureCheckModel) { + if openAIReq.ReasoningEffort != "" && effortAvailable(featureCheckModel, hints) { effort, effortErr := mapReasoningEffortToOutputConfigEffort(openAIReq.ReasoningEffort) if effortErr != nil { return nil, effortErr diff --git a/internal/translator/anthropic_helper_test.go b/internal/translator/anthropic_helper_test.go index 5ab077e619..1173c00a2a 100644 --- a/internal/translator/anthropic_helper_test.go +++ b/internal/translator/anthropic_helper_test.go @@ -852,7 +852,7 @@ func TestOutputConfigAvailable(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := outputConfigAvailable(tt.apiSchema, tt.model) + result := outputConfigAvailable(tt.apiSchema, tt.model, nil) require.Equal(t, tt.expected, result) }) } @@ -1250,12 +1250,59 @@ func TestEffortAvailable(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := effortAvailable(tt.model) + result := effortAvailable(tt.model, nil) require.Equal(t, tt.expected, result) }) } } +// TestOutputConfigAvailableWithHints verifies that translation hints override the model-name heuristic. +func TestOutputConfigAvailableWithHints(t *testing.T) { + // Hint true forces support even for a model the heuristic would reject. + require.True(t, outputConfigAvailable(filterapi.APISchemaAWSAnthropic, "claude-3-sonnet", &filterapi.ModelTranslationHints{SupportsOutputConfig: ptr.To(true)})) + // Hint false forces no support even for a model the heuristic would accept. + require.False(t, outputConfigAvailable(filterapi.APISchemaAWSAnthropic, "claude-opus-4-6-20250514", &filterapi.ModelTranslationHints{SupportsOutputConfig: ptr.To(false)})) + // Nil field falls back to the heuristic. + require.True(t, outputConfigAvailable(filterapi.APISchemaAWSAnthropic, "claude-opus-4-6-20250514", &filterapi.ModelTranslationHints{})) +} + +// TestEffortAvailableWithHints verifies that translation hints override the model-name heuristic. +func TestEffortAvailableWithHints(t *testing.T) { + require.True(t, effortAvailable("claude-3-sonnet", &filterapi.ModelTranslationHints{SupportsReasoningEffort: ptr.To(true)})) + require.False(t, effortAvailable("claude-opus-4-6-20250514", &filterapi.ModelTranslationHints{SupportsReasoningEffort: ptr.To(false)})) + require.True(t, effortAvailable("claude-opus-4-6-20250514", &filterapi.ModelTranslationHints{})) +} + +// TestBuildAnthropicParamsMaxTokensFromHints verifies max_tokens defaulting behavior when the client omits it. +func TestBuildAnthropicParamsMaxTokensFromHints(t *testing.T) { + newReq := func() *openai.ChatCompletionRequest { + return &openai.ChatCompletionRequest{ + Model: "claude-opus-4-6", + Messages: []openai.ChatCompletionMessageParamUnion{{OfUser: &openai.ChatCompletionUserMessageParam{Content: openai.StringOrUserRoleContentUnion{Value: "hi"}}}}, + } + } + + t.Run("client omits max_tokens, hint provides default", func(t *testing.T) { + params, err := buildAnthropicParams(newReq(), filterapi.APISchemaAWSAnthropic, "", &filterapi.ModelTranslationHints{MaxOutputTokens: ptr.To(int64(32000))}) + require.NoError(t, err) + require.Equal(t, int64(32000), params.MaxTokens) + }) + + t.Run("client max_tokens takes precedence over hint", func(t *testing.T) { + req := newReq() + req.MaxTokens = ptr.To(int64(100)) + params, err := buildAnthropicParams(req, filterapi.APISchemaAWSAnthropic, "", &filterapi.ModelTranslationHints{MaxOutputTokens: ptr.To(int64(32000))}) + require.NoError(t, err) + require.Equal(t, int64(100), params.MaxTokens) + }) + + t.Run("no hint and no client value keeps existing behavior (0)", func(t *testing.T) { + params, err := buildAnthropicParams(newReq(), filterapi.APISchemaAWSAnthropic, "", nil) + require.NoError(t, err) + require.Equal(t, int64(0), params.MaxTokens) + }) +} + func TestBuildAnthropicParamsWithStructuredOutput(t *testing.T) { tests := []struct { name string @@ -1451,7 +1498,7 @@ func TestBuildAnthropicParamsWithStructuredOutput(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - params, err := buildAnthropicParams(tt.request, tt.apiSchema, "") + params, err := buildAnthropicParams(tt.request, tt.apiSchema, "", nil) if tt.expectErr { require.Error(t, err) @@ -1492,7 +1539,7 @@ func TestBuildAnthropicParamsWithStructuredOutput(t *testing.T) { }, } // The modelNameOverride contains a recognized model identifier. - params, err := buildAnthropicParams(request, filterapi.APISchemaAWSAnthropic, "us.anthropic.claude-sonnet-4-5-20250514-v1:0") + params, err := buildAnthropicParams(request, filterapi.APISchemaAWSAnthropic, "us.anthropic.claude-sonnet-4-5-20250514-v1:0", nil) require.NoError(t, err) require.NotNil(t, params) require.NotNil(t, params.OutputConfig.Format.Schema) @@ -1522,7 +1569,7 @@ func TestBuildAnthropicParamsPreservesStructuredOutputPropertyOrder(t *testing.T }, } - params, err := buildAnthropicParams(request, filterapi.APISchemaAWSAnthropic, "") + params, err := buildAnthropicParams(request, filterapi.APISchemaAWSAnthropic, "", nil) require.NoError(t, err) body, err := json.Marshal(params) @@ -1659,7 +1706,7 @@ func TestBuildAnthropicParamsWithReasoningEffort(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - params, err := buildAnthropicParams(tt.request, filterapi.APISchemaAWSAnthropic, "") + params, err := buildAnthropicParams(tt.request, filterapi.APISchemaAWSAnthropic, "", nil) require.NoError(t, err) require.NotNil(t, params) require.Equal(t, tt.expectedEffort, params.OutputConfig.Effort) @@ -1678,7 +1725,7 @@ func TestBuildAnthropicParamsWithReasoningEffort(t *testing.T) { }}, }, } - _, err := buildAnthropicParams(request, filterapi.APISchemaAWSAnthropic, "") + _, err := buildAnthropicParams(request, filterapi.APISchemaAWSAnthropic, "", nil) require.Error(t, err) require.ErrorIs(t, err, internalapi.ErrInvalidRequestBody) require.Contains(t, err.Error(), "unsupported reasoning effort level") @@ -1697,7 +1744,7 @@ func TestBuildAnthropicParamsWithReasoningEffort(t *testing.T) { }, } // The modelNameOverride contains a recognized model identifier. - params, err := buildAnthropicParams(request, filterapi.APISchemaAWSAnthropic, "us.anthropic.claude-opus-4-5-20250514-v1:0") + params, err := buildAnthropicParams(request, filterapi.APISchemaAWSAnthropic, "us.anthropic.claude-opus-4-5-20250514-v1:0", nil) require.NoError(t, err) require.NotNil(t, params) require.Equal(t, anthropic.OutputConfigEffortHigh, params.OutputConfig.Effort) @@ -1716,7 +1763,7 @@ func TestBuildAnthropicParamsWithReasoningEffort(t *testing.T) { }, } // The modelNameOverride points to an unsupported model. - params, err := buildAnthropicParams(request, filterapi.APISchemaAWSAnthropic, "us.anthropic.claude-3-sonnet-20240229-v1:0") + params, err := buildAnthropicParams(request, filterapi.APISchemaAWSAnthropic, "us.anthropic.claude-3-sonnet-20240229-v1:0", nil) require.NoError(t, err) require.NotNil(t, params) require.Equal(t, anthropic.OutputConfigEffort(""), params.OutputConfig.Effort) diff --git a/internal/translator/gemini_helper.go b/internal/translator/gemini_helper.go index 8ca1ffc550..901f743180 100644 --- a/internal/translator/gemini_helper.go +++ b/internal/translator/gemini_helper.go @@ -20,6 +20,7 @@ import ( "google.golang.org/genai" "github.com/envoyproxy/ai-gateway/internal/apischema/openai" + "github.com/envoyproxy/ai-gateway/internal/filterapi" "github.com/envoyproxy/ai-gateway/internal/internalapi" "github.com/envoyproxy/ai-gateway/internal/json" ) @@ -54,7 +55,7 @@ const ( // -------------------------------------------------------------. // openAIMessagesToGeminiContents converts OpenAI messages to Gemini Contents and SystemInstruction. -func openAIMessagesToGeminiContents(messages []openai.ChatCompletionMessageParamUnion, requestModel internalapi.RequestModel) ([]genai.Content, *genai.Content, error) { +func openAIMessagesToGeminiContents(messages []openai.ChatCompletionMessageParamUnion, requestModel internalapi.RequestModel, hints *filterapi.ModelTranslationHints) ([]genai.Content, *genai.Content, error) { var gcpContents []genai.Content var systemInstruction *genai.Content knownToolCalls := make(map[string]string) @@ -89,7 +90,7 @@ func openAIMessagesToGeminiContents(messages []openai.ChatCompletionMessageParam } case msgUnion.OfUser != nil: msg := msgUnion.OfUser - parts, err := userMsgToGeminiParts(*msg, requestModel) + parts, err := userMsgToGeminiParts(*msg, requestModel, hints) if err != nil { return nil, nil, fmt.Errorf("invalid user message: %w", err) } @@ -170,7 +171,7 @@ func developerMsgToGeminiParts(msg openai.ChatCompletionDeveloperMessageParam) ( } // userMsgToGeminiParts converts OpenAI user message to Gemini Parts. -func userMsgToGeminiParts(msg openai.ChatCompletionUserMessageParam, requestModel internalapi.RequestModel) ([]*genai.Part, error) { +func userMsgToGeminiParts(msg openai.ChatCompletionUserMessageParam, requestModel internalapi.RequestModel, hints *filterapi.ModelTranslationHints) ([]*genai.Part, error) { var parts []*genai.Part switch contentValue := msg.Content.Value.(type) { case string: @@ -211,7 +212,7 @@ func userMsgToGeminiParts(msg openai.ChatCompletionUserMessageParam, requestMode } // Handle media resolution for both data URI and regular URL cases - if content.OfImageURL.ImageURL.Detail != "" && mediaResolutionAvailable(requestModel) { + if content.OfImageURL.ImageURL.Detail != "" && mediaResolutionAvailable(requestModel, hints) { mediaResolution, err := mapDetailMediaResolution(content.OfImageURL.ImageURL.Detail) if err != nil { return nil, fmt.Errorf("%w: invalid detail", internalapi.ErrInvalidRequestBody) @@ -581,20 +582,35 @@ func openAIToolChoiceToGeminiToolConfig(toolChoice *openai.ChatCompletionToolCho // Check and Update these functions when new Gemini model versions are released // and their feature support changes. -// it only works with models after gemini2.5 according to https://ai.google.dev/gemini-api/docs/structured-output#json-schema, separate it as a small function to make it easier to maintain -func responseJSONSchemaAvailable(requestModel internalapi.RequestModel) bool { +// responseJSONSchemaAvailable reports whether the model accepts a JSON Schema in response_format. +// When hints declare SupportsResponseJSONSchema, that value is authoritative; otherwise it falls back +// to the model-name heuristic (JSON schema only works with Gemini 2.5+ per +// https://ai.google.dev/gemini-api/docs/structured-output#json-schema). +func responseJSONSchemaAvailable(requestModel internalapi.RequestModel, hints *filterapi.ModelTranslationHints) bool { + if hints != nil && hints.SupportsResponseJSONSchema != nil { + return *hints.SupportsResponseJSONSchema + } return strings.Contains(requestModel, "gemini") && (strings.Contains(requestModel, "2.5") || strings.Contains(requestModel, "3")) } // mediaResolutionAvailable checks if the model supports media resolution settings. -// Only Gemini 3.0+ models support this feature for controlling image/video quality. -func mediaResolutionAvailable(requestModel internalapi.RequestModel) bool { +// When hints declare SupportsReasoningEffort, that value is authoritative (media resolution is gated +// by the same Gemini 3.0+ generation as reasoning effort); otherwise it falls back to the model-name +// heuristic. Only Gemini 3.0+ models support this feature for controlling image/video quality. +func mediaResolutionAvailable(requestModel internalapi.RequestModel, hints *filterapi.ModelTranslationHints) bool { + if hints != nil && hints.SupportsReasoningEffort != nil { + return *hints.SupportsReasoningEffort + } return strings.Contains(requestModel, "gemini") && strings.Contains(requestModel, "3") } // reasoningEffortAvailable checks if the model supports reasoning effort. -// Only Gemini 3.0+ models support this feature for controlling reasoning depth. -func reasoningEffortAvailable(requestModel internalapi.RequestModel) bool { +// When hints declare SupportsReasoningEffort, that value is authoritative; otherwise it falls back to +// the model-name heuristic. Only Gemini 3.0+ models support this feature for controlling reasoning depth. +func reasoningEffortAvailable(requestModel internalapi.RequestModel, hints *filterapi.ModelTranslationHints) bool { + if hints != nil && hints.SupportsReasoningEffort != nil { + return *hints.SupportsReasoningEffort + } return strings.Contains(requestModel, "gemini") && strings.Contains(requestModel, "3") } @@ -637,7 +653,7 @@ func mapReasoningEffortToThinkingLevel(reasonEffort openai.ReasoningEffort, mode } // openAIReqToGeminiGenerationConfig converts OpenAI request to Gemini GenerationConfig. -func openAIReqToGeminiGenerationConfig(openAIReq *openai.ChatCompletionRequest, requestModel internalapi.RequestModel) (*genai.GenerationConfig, geminiResponseMode, error) { +func openAIReqToGeminiGenerationConfig(openAIReq *openai.ChatCompletionRequest, requestModel internalapi.RequestModel, hints *filterapi.ModelTranslationHints) (*genai.GenerationConfig, geminiResponseMode, error) { responseMode := responseModeNone gc := &genai.GenerationConfig{} if openAIReq.Temperature != nil { @@ -683,7 +699,7 @@ func openAIReqToGeminiGenerationConfig(openAIReq *openai.ChatCompletionRequest, responseMode = responseModeJSON - if responseJSONSchemaAvailable(requestModel) { + if responseJSONSchemaAvailable(requestModel, hints) { gc.ResponseJsonSchema = schemaMap } else { convertedSchema, err := jsonSchemaToGemini(schemaMap) @@ -725,7 +741,7 @@ func openAIReqToGeminiGenerationConfig(openAIReq *openai.ChatCompletionRequest, gc.ResponseMIMEType = mimeTypeApplicationJSON gc.ResponseJsonSchema = openAIReq.GuidedJSON } - if openAIReq.ReasoningEffort != "" && reasoningEffortAvailable(requestModel) { + if openAIReq.ReasoningEffort != "" && reasoningEffortAvailable(requestModel, hints) { thinkLevel, err := mapReasoningEffortToThinkingLevel(openAIReq.ReasoningEffort, requestModel) if err != nil { return nil, responseMode, fmt.Errorf("invalid reasoning effort: %w", err) diff --git a/internal/translator/gemini_helper_test.go b/internal/translator/gemini_helper_test.go index f527f7682b..5e151585c5 100644 --- a/internal/translator/gemini_helper_test.go +++ b/internal/translator/gemini_helper_test.go @@ -20,10 +20,34 @@ import ( "k8s.io/utils/ptr" "github.com/envoyproxy/ai-gateway/internal/apischema/openai" + "github.com/envoyproxy/ai-gateway/internal/filterapi" "github.com/envoyproxy/ai-gateway/internal/internalapi" "github.com/envoyproxy/ai-gateway/internal/json" ) +// TestGeminiFeatureGatesWithHints verifies that translation hints override the model-name heuristics. +func TestGeminiFeatureGatesWithHints(t *testing.T) { + t.Run("responseJSONSchemaAvailable", func(t *testing.T) { + // Hint true forces support even for a model the heuristic would reject (e.g. gemini-4). + require.True(t, responseJSONSchemaAvailable("gemini-4-pro", &filterapi.ModelTranslationHints{SupportsResponseJSONSchema: ptr.To(true)})) + // Hint false forces no support even for a model the heuristic would accept. + require.False(t, responseJSONSchemaAvailable("gemini-2.5-pro", &filterapi.ModelTranslationHints{SupportsResponseJSONSchema: ptr.To(false)})) + // Nil field falls back to the heuristic. + require.True(t, responseJSONSchemaAvailable("gemini-2.5-pro", &filterapi.ModelTranslationHints{})) + require.False(t, responseJSONSchemaAvailable("gemini-4-pro", nil)) + }) + t.Run("reasoningEffortAvailable", func(t *testing.T) { + require.True(t, reasoningEffortAvailable("gemini-4-pro", &filterapi.ModelTranslationHints{SupportsReasoningEffort: ptr.To(true)})) + require.False(t, reasoningEffortAvailable("gemini-3-pro", &filterapi.ModelTranslationHints{SupportsReasoningEffort: ptr.To(false)})) + require.True(t, reasoningEffortAvailable("gemini-3-pro", nil)) + }) + t.Run("mediaResolutionAvailable", func(t *testing.T) { + require.True(t, mediaResolutionAvailable("gemini-4-pro", &filterapi.ModelTranslationHints{SupportsReasoningEffort: ptr.To(true)})) + require.False(t, mediaResolutionAvailable("gemini-3-pro", &filterapi.ModelTranslationHints{SupportsReasoningEffort: ptr.To(false)})) + require.True(t, mediaResolutionAvailable("gemini-3-pro", nil)) + }) +} + func TestOpenAIMessagesToGeminiContents(t *testing.T) { tests := []struct { name string @@ -124,7 +148,7 @@ func TestOpenAIMessagesToGeminiContents(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - contents, systemInstruction, err := openAIMessagesToGeminiContents(tc.messages, "gemini-3-pro") + contents, systemInstruction, err := openAIMessagesToGeminiContents(tc.messages, "gemini-3-pro", nil) if tc.expectedErrorMsg != "" || err != nil { require.Error(t, err) @@ -1227,7 +1251,7 @@ func TestUserMsgToGeminiParts(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - parts, err := userMsgToGeminiParts(tc.msg, "gemini-3-pro") + parts, err := userMsgToGeminiParts(tc.msg, "gemini-3-pro", nil) if tc.expectedErrMsg != "" || err != nil { require.Error(t, err) @@ -1525,7 +1549,7 @@ func TestOpenAIReqToGeminiGenerationConfig(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - got, responseMode, err := openAIReqToGeminiGenerationConfig(tc.input, tc.requestModel) + got, responseMode, err := openAIReqToGeminiGenerationConfig(tc.input, tc.requestModel, nil) if tc.expectedErrMsg != "" { require.ErrorContains(t, err, tc.expectedErrMsg) } else { @@ -3252,7 +3276,7 @@ func TestOpenAIReqToGeminiGenerationConfigWithJsonSchemaToGemini(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - got, responseMode, err := openAIReqToGeminiGenerationConfig(tc.input, tc.requestModel) + got, responseMode, err := openAIReqToGeminiGenerationConfig(tc.input, tc.requestModel, nil) if tc.expectedErrMsg != "" { require.ErrorContains(t, err, tc.expectedErrMsg) } else { diff --git a/internal/translator/openai_awsanthropic.go b/internal/translator/openai_awsanthropic.go index 2b5d53f5ea..b948741705 100644 --- a/internal/translator/openai_awsanthropic.go +++ b/internal/translator/openai_awsanthropic.go @@ -33,10 +33,11 @@ const BedrockDefaultVersion = "bedrock-2023-05-31" // NewChatCompletionOpenAIToAWSAnthropicTranslator implements [Factory] for OpenAI to AWS Anthropic translation. // This translator converts OpenAI ChatCompletion API requests to AWS Anthropic API format. -func NewChatCompletionOpenAIToAWSAnthropicTranslator(apiVersion string, modelNameOverride internalapi.ModelNameOverride) OpenAIChatCompletionTranslator { +func NewChatCompletionOpenAIToAWSAnthropicTranslator(apiVersion string, modelNameOverride internalapi.ModelNameOverride, hints *filterapi.ModelTranslationHints) OpenAIChatCompletionTranslator { return &openAIToAWSAnthropicTranslatorV1ChatCompletion{ apiVersion: apiVersion, modelNameOverride: modelNameOverride, + hints: hints, } } @@ -46,6 +47,7 @@ func NewChatCompletionOpenAIToAWSAnthropicTranslator(apiVersion string, modelNam type openAIToAWSAnthropicTranslatorV1ChatCompletion struct { apiVersion string modelNameOverride internalapi.ModelNameOverride + hints *filterapi.ModelTranslationHints streamParser *anthropicStreamParser requestModel internalapi.RequestModel bufferedBody []byte @@ -71,7 +73,7 @@ func (o *openAIToAWSAnthropicTranslatorV1ChatCompletion) RequestBody(_ []byte, o o.streamParser = newAnthropicStreamParser(o.requestModel) } - params, err := buildAnthropicParams(openAIReq, filterapi.APISchemaAWSAnthropic, o.modelNameOverride) + params, err := buildAnthropicParams(openAIReq, filterapi.APISchemaAWSAnthropic, o.modelNameOverride, o.hints) if err != nil { return } diff --git a/internal/translator/openai_awsanthropic_test.go b/internal/translator/openai_awsanthropic_test.go index 44762b448f..96f9e29e25 100644 --- a/internal/translator/openai_awsanthropic_test.go +++ b/internal/translator/openai_awsanthropic_test.go @@ -111,7 +111,7 @@ func wrapAnthropicSSEInEventStream(sseData string) ([]byte, error) { // AWS Anthropic uses deterministic model mapping without virtualization func TestResponseModel_AWSAnthropic(t *testing.T) { modelName := "anthropic.claude-sonnet-4-20250514-v1:0" - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", modelName) + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", modelName, nil) // Initialize translator with the model req := &openai.ChatCompletionRequest{ @@ -179,7 +179,7 @@ func TestOpenAIToAWSAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T } t.Run("AWS Bedrock InvokeModel Values Configured Correctly", func(t *testing.T) { - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "", nil) hm, body, err := translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) require.NotNil(t, hm) @@ -202,7 +202,7 @@ func TestOpenAIToAWSAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T t.Run("Model Name Override", func(t *testing.T) { overrideModelName := "anthropic.claude-3-haiku-20240307-v1:0" // Instantiate the translator with the model name override. - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", overrideModelName) + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", overrideModelName, nil) // Call RequestBody with the original request, which has a different model name. hm, _, err := translator.RequestBody(nil, openAIReq, false) @@ -218,7 +218,7 @@ func TestOpenAIToAWSAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T t.Run("Model Name with ARN (URL encoding)", func(t *testing.T) { arnModelName := "arn:aws:bedrock:us-east-1::foundation-model/anthropic.claude-3-opus-20240229-v1:0" - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", arnModelName) + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", arnModelName, nil) hm, _, err := translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -240,7 +240,7 @@ func TestOpenAIToAWSAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T MaxTokens: ptr.To(int64(100)), Stream: true, } - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "", nil) hm, body, err := translator.RequestBody(nil, streamReq, false) require.NoError(t, err) require.NotNil(t, hm) @@ -261,7 +261,7 @@ func TestOpenAIToAWSAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T t.Run("API Version Override", func(t *testing.T) { customAPIVersion := "bedrock-2024-01-01" // Instantiate the translator with the custom API version. - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator(customAPIVersion, "") + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator(customAPIVersion, "", nil) // Call RequestBody with a standard request. _, body, err := translator.RequestBody(nil, openAIReq, false) @@ -279,7 +279,7 @@ func TestOpenAIToAWSAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T MaxTokens: ptr.To(int64(100)), Temperature: ptr.To(2.5), } - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "", nil) _, _, err := translator.RequestBody(nil, invalidTempReq, false) require.Error(t, err) require.Contains(t, err.Error(), fmt.Sprintf(tempNotSupportedError, *invalidTempReq.Temperature)) @@ -290,7 +290,7 @@ func TestOpenAIToAWSAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T Model: "anthropic.claude-3-opus-20240229-v1:0", Messages: []openai.ChatCompletionMessageParamUnion{}, } - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, missingTokensReq, false) require.NoError(t, err) require.Equal(t, int64(0), gjson.GetBytes(body, "max_tokens").Int()) @@ -299,7 +299,7 @@ func TestOpenAIToAWSAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T func TestOpenAIToAWSAnthropicTranslatorV1ChatCompletion_ResponseBody(t *testing.T) { t.Run("invalid json body", func(t *testing.T) { - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "", nil) _, _, _, _, err := translator.ResponseBody(map[string]string{statusHeaderName: "200"}, bytes.NewBufferString("invalid json"), true, nil) require.Error(t, err) require.Contains(t, err.Error(), "failed to unmarshal body") @@ -400,7 +400,7 @@ func TestOpenAIToAWSAnthropicTranslatorV1ChatCompletion_ResponseBody(t *testing. body, err := json.Marshal(tt.inputResponse) require.NoError(t, err, "Test setup failed: could not marshal input struct") - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "", nil) hm, body, usedToken, _, err := translator.ResponseBody(tt.respHeaders, bytes.NewBuffer(body), true, nil) require.NoError(t, err, "Translator returned an unexpected internal error") @@ -545,7 +545,7 @@ data: {"type": "message_stop"} MaxTokens: new(int64), } - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "").(*openAIToAWSAnthropicTranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "", nil).(*openAIToAWSAnthropicTranslatorV1ChatCompletion) _, _, err = translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -598,7 +598,7 @@ data: {"type": "message_stop"} Model: "test-model", MaxTokens: new(int64), } - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "").(*openAIToAWSAnthropicTranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "", nil).(*openAIToAWSAnthropicTranslatorV1ChatCompletion) _, _, err = translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -650,7 +650,7 @@ data: {"type": "message_stop"} require.NoError(t, err) openAIReq := &openai.ChatCompletionRequest{Stream: true, Model: "test-model", MaxTokens: new(int64)} - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "").(*openAIToAWSAnthropicTranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "", nil).(*openAIToAWSAnthropicTranslatorV1ChatCompletion) _, _, err = translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -705,7 +705,7 @@ data: {"type":"message_stop"} require.NoError(t, err) openAIReq := &openai.ChatCompletionRequest{Stream: true, Model: "test-model", MaxTokens: new(int64)} - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "").(*openAIToAWSAnthropicTranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "", nil).(*openAIToAWSAnthropicTranslatorV1ChatCompletion) _, _, err = translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -728,7 +728,7 @@ func TestAWSAnthropicStreamParser_ErrorHandling(t *testing.T) { require.NoError(t, err) openAIReq := &openai.ChatCompletionRequest{Stream: true, Model: "test-model", MaxTokens: new(int64)} - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "").(*openAIToAWSAnthropicTranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", "", nil).(*openAIToAWSAnthropicTranslatorV1ChatCompletion) _, _, err = translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -801,7 +801,7 @@ func TestOpenAIToAWSAnthropicTranslator_EdgeCases(t *testing.T) { t.Run("response with model field from API", func(t *testing.T) { // AWS Anthropic may return model field in response modelName := "custom-override-model" - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", modelName) + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", modelName, nil) req := &openai.ChatCompletionRequest{ Model: "original-model", @@ -838,7 +838,7 @@ func TestOpenAIToAWSAnthropicTranslator_EdgeCases(t *testing.T) { t.Run("response without model field", func(t *testing.T) { // AWS Anthropic typically doesn't return model field modelName := "anthropic.claude-3-haiku-20240307-v1:0" - translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", modelName) + translator := NewChatCompletionOpenAIToAWSAnthropicTranslator("", modelName, nil) req := &openai.ChatCompletionRequest{ Model: "original-model", diff --git a/internal/translator/openai_gcpanthropic.go b/internal/translator/openai_gcpanthropic.go index 39bf39bb2a..b22fc1222a 100644 --- a/internal/translator/openai_gcpanthropic.go +++ b/internal/translator/openai_gcpanthropic.go @@ -32,10 +32,11 @@ const ( // NewChatCompletionOpenAIToGCPAnthropicTranslator implements [Factory] for OpenAI to GCP Anthropic translation. // This translator converts OpenAI ChatCompletion API requests to GCP Anthropic API format. -func NewChatCompletionOpenAIToGCPAnthropicTranslator(apiVersion string, modelNameOverride internalapi.ModelNameOverride) OpenAIChatCompletionTranslator { +func NewChatCompletionOpenAIToGCPAnthropicTranslator(apiVersion string, modelNameOverride internalapi.ModelNameOverride, hints *filterapi.ModelTranslationHints) OpenAIChatCompletionTranslator { return &openAIToGCPAnthropicTranslatorV1ChatCompletion{ apiVersion: apiVersion, modelNameOverride: modelNameOverride, + hints: hints, } } @@ -45,6 +46,7 @@ func NewChatCompletionOpenAIToGCPAnthropicTranslator(apiVersion string, modelNam type openAIToGCPAnthropicTranslatorV1ChatCompletion struct { apiVersion string modelNameOverride internalapi.ModelNameOverride + hints *filterapi.ModelTranslationHints streamParser *anthropicStreamParser requestModel internalapi.RequestModel // Redaction configuration for debug logging @@ -57,7 +59,7 @@ type openAIToGCPAnthropicTranslatorV1ChatCompletion struct { func (o *openAIToGCPAnthropicTranslatorV1ChatCompletion) RequestBody(_ []byte, openAIReq *openai.ChatCompletionRequest, _ bool) ( newHeaders []internalapi.Header, newBody []byte, err error, ) { - params, err := buildAnthropicParams(openAIReq, filterapi.APISchemaGCPAnthropic, o.modelNameOverride) + params, err := buildAnthropicParams(openAIReq, filterapi.APISchemaGCPAnthropic, o.modelNameOverride, o.hints) if err != nil { return } diff --git a/internal/translator/openai_gcpanthropic_test.go b/internal/translator/openai_gcpanthropic_test.go index ce2d7c7e59..bb4360d72c 100644 --- a/internal/translator/openai_gcpanthropic_test.go +++ b/internal/translator/openai_gcpanthropic_test.go @@ -42,7 +42,7 @@ const ( // GCP Anthropic uses deterministic model mapping without virtualization func TestResponseModel_GCPAnthropic(t *testing.T) { modelName := "claude-sonnet-4@20250514" - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", modelName) + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", modelName, nil) // Initialize translator with the model req := &openai.ChatCompletionRequest{ @@ -109,7 +109,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T Temperature: ptr.To(0.7), } t.Run("Vertex Values Configured Correctly", func(t *testing.T) { - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) hm, body, err := translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) require.NotNil(t, hm) @@ -133,7 +133,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T t.Run("Model Name Override", func(t *testing.T) { overrideModelName := "claude-3" // Instantiate the translator with the model name override. - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", overrideModelName) + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", overrideModelName, nil) // Call RequestBody with the original request, which has a different model name. hm, _, err := translator.RequestBody(nil, openAIReq, false) @@ -169,7 +169,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T }, }, } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, imageReq, false) require.NoError(t, err) @@ -193,7 +193,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T }, MaxTokens: ptr.To(int64(100)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, multiSystemReq, false) require.NoError(t, err) @@ -209,7 +209,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T MaxTokens: ptr.To(int64(100)), Stream: true, } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) hm, body, err := translator.RequestBody(nil, streamReq, false) require.NoError(t, err) require.NotNil(t, hm) @@ -234,7 +234,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T OfStringArray: []string{"stop1", "stop2"}, }, } - messageParam, err := buildAnthropicParams(openaiRequest, filterapi.APISchemaGCPAnthropic, "") + messageParam, err := buildAnthropicParams(openaiRequest, filterapi.APISchemaGCPAnthropic, "", nil) require.NoError(t, err) require.Equal(t, int64(100), messageParam.MaxTokens) require.Equal(t, "0.1", messageParam.TopP.String()) @@ -253,7 +253,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T OfString: openaigo.Opt[string]("stop1"), }, } - messageParam, err := buildAnthropicParams(openaiRequest, filterapi.APISchemaGCPAnthropic, "") + messageParam, err := buildAnthropicParams(openaiRequest, filterapi.APISchemaGCPAnthropic, "", nil) require.NoError(t, err) require.Equal(t, int64(100), messageParam.MaxTokens) require.Equal(t, "0.1", messageParam.TopP.String()) @@ -268,7 +268,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T MaxTokens: ptr.To(int64(100)), Temperature: ptr.To(2.5), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, _, err := translator.RequestBody(nil, invalidTempReq, false) require.ErrorIs(t, err, internalapi.ErrInvalidRequestBody) }) @@ -280,7 +280,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T MaxTokens: ptr.To(int64(100)), Temperature: ptr.To(-2.5), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, _, err := translator.RequestBody(nil, invalidTempReq, false) require.ErrorIs(t, err, internalapi.ErrInvalidRequestBody) }) @@ -290,7 +290,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T Model: claudeTestModel, Messages: []openai.ChatCompletionMessageParamUnion{}, } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, missingTokensReq, false) require.NoError(t, err) require.Equal(t, int64(0), gjson.GetBytes(body, "max_tokens").Int()) @@ -298,7 +298,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T t.Run("API Version Override", func(t *testing.T) { customAPIVersion := "bedrock-2023-05-31" // Instantiate the translator with the custom API version. - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator(customAPIVersion, "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator(customAPIVersion, "", nil) // Call RequestBody with a standard request. _, body, err := translator.RequestBody(nil, openAIReq, false) @@ -322,7 +322,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T }, }, } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, thinkingReq, false) require.NoError(t, err) require.NotNil(t, body) @@ -345,7 +345,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T }, }, } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, thinkingReq, false) require.NoError(t, err) require.NotNil(t, body) @@ -370,7 +370,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T }, }, } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, thinkingReq, false) require.NoError(t, err) require.NotNil(t, body) @@ -393,7 +393,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T }, }, } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, thinkingReq, false) require.NoError(t, err) require.NotNil(t, body) @@ -408,7 +408,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_ResponseBody(t *testing.T) { t.Run("invalid json body", func(t *testing.T) { - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, _, _, _, err := translator.ResponseBody(map[string]string{statusHeaderName: "200"}, bytes.NewBufferString("invalid json"), true, nil) require.Error(t, err) require.Contains(t, err.Error(), "failed to unmarshal body") @@ -636,7 +636,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_ResponseBody(t *testing. body, err := json.Marshal(tt.inputResponse) require.NoError(t, err, "Test setup failed: could not marshal input struct") - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) hm, body, usedToken, _, err := translator.ResponseBody(tt.respHeaders, bytes.NewBuffer(body), true, nil) require.NoError(t, err, "Translator returned an unexpected internal error") @@ -1341,7 +1341,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_Cache(t *testing.T) { MaxTokens: ptr.To(int64(100)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -1438,7 +1438,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_Cache(t *testing.T) { MaxTokens: ptr.To(int64(10)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, req, false) require.NoError(t, err) @@ -1484,7 +1484,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_Cache(t *testing.T) { MaxTokens: ptr.To(int64(50)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, req, false) require.NoError(t, err) @@ -1528,7 +1528,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_Cache(t *testing.T) { MaxTokens: ptr.To(int64(50)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, req, false) require.NoError(t, err) @@ -1562,7 +1562,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_Cache(t *testing.T) { MaxTokens: ptr.To(int64(100)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -1602,7 +1602,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_Cache(t *testing.T) { MaxTokens: ptr.To(int64(100)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -1647,7 +1647,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_Cache(t *testing.T) { MaxTokens: ptr.To(int64(100)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -1691,7 +1691,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_Cache(t *testing.T) { MaxTokens: ptr.To(int64(100)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -1733,7 +1733,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_Cache(t *testing.T) { MaxTokens: ptr.To(int64(100)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -1770,7 +1770,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_Cache(t *testing.T) { MaxTokens: ptr.To(int64(100)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -1824,7 +1824,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_Cache(t *testing.T) { MaxTokens: ptr.To(int64(100)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "") + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil) _, body, err := translator.RequestBody(nil, openAIReq, false) require.NoError(t, err) @@ -1842,7 +1842,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_Cache(t *testing.T) { } func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_SetRedactionConfig(t *testing.T) { - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "").(*openAIToGCPAnthropicTranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil).(*openAIToGCPAnthropicTranslatorV1ChatCompletion) logger := slog.New(slog.NewTextHandler(io.Discard, nil)) translator.SetRedactionConfig(true, true, logger) @@ -1853,7 +1853,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_SetRedactionConfig(t *te } func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RedactBody(t *testing.T) { - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "").(*openAIToGCPAnthropicTranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil).(*openAIToGCPAnthropicTranslatorV1ChatCompletion) t.Run("nil response returns nil", func(t *testing.T) { result := translator.RedactBody(nil) @@ -1991,7 +1991,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_ResponseHeaders(t *testi Model: "test-model", MaxTokens: ptr.To(int64(100)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "").(*openAIToGCPAnthropicTranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil).(*openAIToGCPAnthropicTranslatorV1ChatCompletion) // Initialize the stream parser by calling RequestBody with streaming request _, _, err := translator.RequestBody(nil, openAIReq, false) @@ -2011,7 +2011,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_ResponseHeaders(t *testi Model: "test-model", MaxTokens: ptr.To(int64(100)), } - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "").(*openAIToGCPAnthropicTranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil).(*openAIToGCPAnthropicTranslatorV1ChatCompletion) // Initialize without streaming _, _, err := translator.RequestBody(nil, openAIReq, false) @@ -2029,7 +2029,7 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_ResponseBody_WithDebugLo var logBuf bytes.Buffer logger := slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelDebug})) - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "").(*openAIToGCPAnthropicTranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil).(*openAIToGCPAnthropicTranslatorV1ChatCompletion) translator.SetRedactionConfig(true, true, logger) // Initialize translator with the model @@ -2091,7 +2091,7 @@ func (m *mockSpan) EndSpanOnError(_ int, _ []byte) {} func (m *mockSpan) EndSpan() {} func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_ResponseBody_WithSpanRecording(t *testing.T) { - translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "").(*openAIToGCPAnthropicTranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "", nil).(*openAIToGCPAnthropicTranslatorV1ChatCompletion) // Initialize translator with the model req := &openai.ChatCompletionRequest{ diff --git a/internal/translator/openai_gcpvertexai.go b/internal/translator/openai_gcpvertexai.go index 529f265ab8..37028c3493 100644 --- a/internal/translator/openai_gcpvertexai.go +++ b/internal/translator/openai_gcpvertexai.go @@ -20,6 +20,7 @@ import ( "github.com/envoyproxy/ai-gateway/internal/apischema/gcp" "github.com/envoyproxy/ai-gateway/internal/apischema/openai" + "github.com/envoyproxy/ai-gateway/internal/filterapi" "github.com/envoyproxy/ai-gateway/internal/internalapi" "github.com/envoyproxy/ai-gateway/internal/json" "github.com/envoyproxy/ai-gateway/internal/metrics" @@ -67,8 +68,8 @@ type gcpVertexAIErrorDetails struct { // NewChatCompletionOpenAIToGCPVertexAITranslator implements [Factory] for OpenAI to GCP Gemini translation. // This translator converts OpenAI ChatCompletion API requests to GCP Gemini API format. -func NewChatCompletionOpenAIToGCPVertexAITranslator(modelNameOverride internalapi.ModelNameOverride) OpenAIChatCompletionTranslator { - return &openAIToGCPVertexAITranslatorV1ChatCompletion{modelNameOverride: modelNameOverride, toolCallIndex: int64(0)} +func NewChatCompletionOpenAIToGCPVertexAITranslator(modelNameOverride internalapi.ModelNameOverride, hints *filterapi.ModelTranslationHints) OpenAIChatCompletionTranslator { + return &openAIToGCPVertexAITranslatorV1ChatCompletion{modelNameOverride: modelNameOverride, hints: hints, toolCallIndex: int64(0)} } // openAIToGCPVertexAITranslatorV1ChatCompletion translates OpenAI Chat Completions API to GCP Vertex AI Gemini API. @@ -77,6 +78,7 @@ func NewChatCompletionOpenAIToGCPVertexAITranslator(modelNameOverride internalap type openAIToGCPVertexAITranslatorV1ChatCompletion struct { responseMode geminiResponseMode modelNameOverride internalapi.ModelNameOverride + hints *filterapi.ModelTranslationHints stream bool // Track if this is a streaming request. streamDelimiter []byte bufferedBody []byte // Buffer for incomplete JSON chunks. @@ -518,13 +520,13 @@ func getGenerationConfigThinkingConfig(tu *openai.ThinkingUnion) *genai.Thinking // openAIMessageToGeminiMessage converts an OpenAI ChatCompletionRequest to a GCP Gemini GenerateContentRequest. func (o *openAIToGCPVertexAITranslatorV1ChatCompletion) openAIMessageToGeminiMessage(openAIReq *openai.ChatCompletionRequest, requestModel internalapi.RequestModel) (*gcp.GenerateContentRequest, error) { // Convert OpenAI messages to Gemini Contents and SystemInstruction. - contents, systemInstruction, err := openAIMessagesToGeminiContents(openAIReq.Messages, requestModel) + contents, systemInstruction, err := openAIMessagesToGeminiContents(openAIReq.Messages, requestModel, o.hints) if err != nil { return nil, err } // Some models support only partialJSONSchema. - parametersJSONSchemaAvailable := responseJSONSchemaAvailable(requestModel) + parametersJSONSchemaAvailable := responseJSONSchemaAvailable(requestModel, o.hints) // Convert OpenAI tools to Gemini tools. tools, err := openAIToolsToGeminiTools(openAIReq.Tools, parametersJSONSchemaAvailable) if err != nil { @@ -538,7 +540,7 @@ func (o *openAIToGCPVertexAITranslatorV1ChatCompletion) openAIMessageToGeminiMes } // Convert generation config. - generationConfig, responseMode, err := openAIReqToGeminiGenerationConfig(openAIReq, requestModel) + generationConfig, responseMode, err := openAIReqToGeminiGenerationConfig(openAIReq, requestModel, o.hints) if err != nil { return nil, fmt.Errorf("invalid generation configs: %w", err) } @@ -577,7 +579,7 @@ func (o *openAIToGCPVertexAITranslatorV1ChatCompletion) applyVendorSpecificField if gcr.GenerationConfig == nil { gcr.GenerationConfig = &genai.GenerationConfig{} } - if vendorGenConfig.MediaResolution != "" && mediaResolutionAvailable(requestModel) { + if vendorGenConfig.MediaResolution != "" && mediaResolutionAvailable(requestModel, o.hints) { gcr.GenerationConfig.MediaResolution = vendorGenConfig.MediaResolution } } diff --git a/internal/translator/openai_gcpvertexai_embeddings.go b/internal/translator/openai_gcpvertexai_embeddings.go index 4badee4049..460e879636 100644 --- a/internal/translator/openai_gcpvertexai_embeddings.go +++ b/internal/translator/openai_gcpvertexai_embeddings.go @@ -198,7 +198,8 @@ func collectPartsFromMessages(messages []openai.ChatCompletionMessageParamUnion, if msg.OfUser == nil { continue } - msgParts, err := userMsgToGeminiParts(*msg.OfUser, requestModel) + // Embeddings do not carry translation hints; pass nil to use the model-name heuristic. + msgParts, err := userMsgToGeminiParts(*msg.OfUser, requestModel, nil) if err != nil { return nil, err } diff --git a/internal/translator/openai_gcpvertexai_test.go b/internal/translator/openai_gcpvertexai_test.go index 439c82d219..a70daeecf1 100644 --- a/internal/translator/openai_gcpvertexai_test.go +++ b/internal/translator/openai_gcpvertexai_test.go @@ -32,7 +32,7 @@ import ( // GCP Vertex AI uses deterministic model mapping without virtualization func TestResponseModel_GCPVertexAIStreaming(t *testing.T) { modelName := "gemini-1.5-pro-002" - translator := NewChatCompletionOpenAIToGCPVertexAITranslator(modelName).(*openAIToGCPVertexAITranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToGCPVertexAITranslator(modelName, nil).(*openAIToGCPVertexAITranslatorV1ChatCompletion) // Initialize translator with streaming request req := &openai.ChatCompletionRequest{ @@ -784,7 +784,7 @@ func TestOpenAIToGCPVertexAITranslatorV1ChatCompletion_RequestBody(t *testing.T) for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - translator := NewChatCompletionOpenAIToGCPVertexAITranslator(tc.modelNameOverride) + translator := NewChatCompletionOpenAIToGCPVertexAITranslator(tc.modelNameOverride, nil) headerMut, bodyMut, err := translator.RequestBody(nil, &tc.input, tc.onRetry) if tc.wantError { assert.Error(t, err) @@ -839,7 +839,7 @@ func TestOpenAIToGCPVertexAITranslatorV1ChatCompletion_ResponseHeaders(t *testin for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - translator := NewChatCompletionOpenAIToGCPVertexAITranslator(tc.modelName) + translator := NewChatCompletionOpenAIToGCPVertexAITranslator(tc.modelName, nil) _, err := translator.ResponseHeaders(tc.headers) if tc.wantError { assert.Error(t, err) @@ -1652,7 +1652,7 @@ func TestExtractToolCallsFromGeminiPartsStream(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - o := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001").(*openAIToGCPVertexAITranslatorV1ChatCompletion) + o := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001", nil).(*openAIToGCPVertexAITranslatorV1ChatCompletion) calls, _, err := o.extractToolCallsFromGeminiPartsStream(toolCalls, tt.input, json.MarshalForDeterministicTesting) if tt.wantErr { @@ -1687,7 +1687,7 @@ func TestExtractToolCallsStreamVsNonStream(t *testing.T) { }, }, } - o := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001").(*openAIToGCPVertexAITranslatorV1ChatCompletion) + o := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001", nil).(*openAIToGCPVertexAITranslatorV1ChatCompletion) // Get results from both functions streamCalls, _, err := o.extractToolCallsFromGeminiPartsStream(toolCallsStream, parts, json.MarshalForDeterministicTesting) @@ -1752,7 +1752,7 @@ func TestExtractToolCallsStreamIndexing(t *testing.T) { }, }, } - o := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001").(*openAIToGCPVertexAITranslatorV1ChatCompletion) + o := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001", nil).(*openAIToGCPVertexAITranslatorV1ChatCompletion) calls, _, err := o.extractToolCallsFromGeminiPartsStream(toolCalls, parts, json.MarshalForDeterministicTesting) require.NoError(t, err) @@ -1785,7 +1785,7 @@ func TestExtractToolCallsStreamIndexing(t *testing.T) { } func TestOpenAIToGCPVertexAITranslatorV1ChatCompletion_StreamingParallelToolIndex(t *testing.T) { - translator := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001").(*openAIToGCPVertexAITranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001", nil).(*openAIToGCPVertexAITranslatorV1ChatCompletion) // Mock multiple GCP streaming response with parallel tool calls gcpToolCallsChunk := `data: { "candidates": [ @@ -1893,7 +1893,7 @@ data: {"candidates": [ // TestOpenAIToGCPVertexAITranslatorV1ChatCompletion_StreamingToolCallWithSignature tests that // streaming tool calls with thought signatures are correctly translated. func TestOpenAIToGCPVertexAITranslatorV1ChatCompletion_StreamingToolCallWithSignature(t *testing.T) { - translator := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001").(*openAIToGCPVertexAITranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001", nil).(*openAIToGCPVertexAITranslatorV1ChatCompletion) // GCP streaming response with thinking followed by tool call with signature gcpStreamingChunk := `data: {"candidates":[{"content":{"parts":[{"text":"let me think about this.", "thought": true}]}}]} @@ -1954,7 +1954,7 @@ func TestOpenAIToGCPVertexAITranslatorV1ChatCompletion_StreamingToolCallSplitFin // must still be "tool_calls", not "stop". (Older Gemini models carried both in // a single chunk; that case is covered by // TestOpenAIToGCPVertexAITranslatorV1ChatCompletion_StreamingToolCallWithSignature.) - translator := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-3.5-flash").(*openAIToGCPVertexAITranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-3.5-flash", nil).(*openAIToGCPVertexAITranslatorV1ChatCompletion) gcpStreamingChunk := `data: {"candidates":[{"content":{"role":"model","parts":[{"functionCall":{"name":"get_weather","args":{"location":"Paris"}},"thoughtSignature":"dG9vbGNhbGxzaWduYXR1cmU="}]}}],"usageMetadata":{"trafficType":"ON_DEMAND"}} @@ -2000,7 +2000,7 @@ data: {"candidates":[{"content":{"role":"model","parts":[{"text":""}]},"finishRe } func TestOpenAIToGCPVertexAITranslatorV1ChatCompletion_StreamingEndOfStream(t *testing.T) { - translator := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001").(*openAIToGCPVertexAITranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001", nil).(*openAIToGCPVertexAITranslatorV1ChatCompletion) // Test end of stream marker. _, body, _, _, err := translator.handleStreamingResponse( @@ -2492,7 +2492,7 @@ Details: [ for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - translator := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001").(*openAIToGCPVertexAITranslatorV1ChatCompletion) + translator := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001", nil).(*openAIToGCPVertexAITranslatorV1ChatCompletion) body := strings.NewReader(tt.body) @@ -2550,7 +2550,7 @@ func bodyMutTransformer(_ *testing.T) cmp.Option { // TestResponseModel_GCPVertexAI tests that GCP Vertex AI returns the request model (no response field) func TestResponseModel_GCPVertexAI(t *testing.T) { modelName := "gemini-1.5-pro-002" - translator := NewChatCompletionOpenAIToGCPVertexAITranslator(modelName) + translator := NewChatCompletionOpenAIToGCPVertexAITranslator(modelName, nil) // Initialize translator with the model req := &openai.ChatCompletionRequest{ diff --git a/internal/translator/tokenize_gcpvertexai.go b/internal/translator/tokenize_gcpvertexai.go index e22dfce61b..7c83f8725b 100644 --- a/internal/translator/tokenize_gcpvertexai.go +++ b/internal/translator/tokenize_gcpvertexai.go @@ -39,7 +39,7 @@ type ToGCPVertexAIV1Tokenize struct { // tokenizeToGeminiCountToken converts an OpenAI tokenize chat request to GCP Gemini CountTokens format. func (o *ToGCPVertexAIV1Tokenize) tokenizeToGeminiCountToken(tokenizeChatReq *tokenize.ChatRequest, requestModel internalapi.RequestModel) (*gcp.CountTokenRequest, error) { // Convert messages to Gemini Contents and SystemInstruction. - contents, systemInstruction, err := openAIMessagesToGeminiContents(tokenizeChatReq.Messages, requestModel) + contents, systemInstruction, err := openAIMessagesToGeminiContents(tokenizeChatReq.Messages, requestModel, nil) if err != nil { return nil, err } @@ -48,7 +48,7 @@ func (o *ToGCPVertexAIV1Tokenize) tokenizeToGeminiCountToken(tokenizeChatReq *to } // Some models support only partialJSONSchema. - parametersJSONSchemaAvailable := responseJSONSchemaAvailable(requestModel) + parametersJSONSchemaAvailable := responseJSONSchemaAvailable(requestModel, nil) // Convert OpenAI tools to Gemini tools. tools, err := openAIToolsToGeminiTools(tokenizeChatReq.Tools, parametersJSONSchemaAvailable) if err != nil { diff --git a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_aiservicebackends.yaml b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_aiservicebackends.yaml index 64a3fa17c1..f4baf0af0a 100644 --- a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_aiservicebackends.yaml +++ b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_aiservicebackends.yaml @@ -310,6 +310,61 @@ spec: - name x-kubernetes-list-type: map type: object + modelTranslationHints: + description: |- + ModelTranslationHints declares translation hints for models routed through this backend, + keyed by the exact ModelNameOverride used in an AIGatewayRouteRuleBackendRef that targets + this backend. This is the single source of truth for a model's hints on this backend: any + AIGatewayRoute, in any namespace, that references this backend with a matching + ModelNameOverride picks up these hints without repeating them. + + Since an AIServiceBackend has exactly one Schema, hints declared here are inherently scoped + to that provider — the same logical model hosted on a different AIServiceBackend (e.g. the + same Claude model on AWS Bedrock vs. GCP Vertex AI) is configured as a separate entry on + that other backend, which is correct because providers can genuinely support different + features for what is nominally the same model. + items: + description: |- + ModelTranslationHintsEntry associates ModelTranslationHints with a specific model routed + through this backend. + properties: + maxOutputTokens: + description: |- + MaxOutputTokens is the maximum number of tokens the model can produce in a single response. + When set, translators use this as the default max_tokens value for APIs that require it + (e.g., the Anthropic Messages API) when the client request omits max_tokens. + format: int64 + minimum: 1 + type: integer + modelNameOverride: + description: |- + ModelNameOverride identifies which model these hints apply to. Must match the + ModelNameOverride configured on an AIGatewayRouteRuleBackendRef that targets this backend + for the hints to take effect. + minLength: 1 + type: string + supportsOutputConfig: + description: |- + SupportsOutputConfig indicates whether the model accepts an Anthropic-style output_config + field for structured output (e.g., Claude Opus/Sonnet 4.5 and later). + type: boolean + supportsReasoningEffort: + description: |- + SupportsReasoningEffort indicates whether the model accepts a reasoning effort parameter + (Gemini thinking level / Anthropic output_config.effort). + type: boolean + supportsResponseJsonSchema: + description: |- + SupportsResponseJSONSchema indicates whether the model accepts a JSON Schema object in + response_format (as opposed to only {"type": "json_object"}). + type: boolean + required: + - modelNameOverride + type: object + type: array + x-kubernetes-list-map-keys: + - modelNameOverride + x-kubernetes-list-type: map schema: description: |- APISchema specifies the API schema of the output format of requests from @@ -779,6 +834,60 @@ spec: type: array x-kubernetes-list-map-keys: - name + modelTranslationHints: + description: |- + ModelTranslationHints declares translation hints for models routed through this backend, + keyed by the exact ModelNameOverride used in an AIGatewayRouteRuleBackendRef that targets + this backend. This is the single source of truth for a model's hints on this backend: any + AIGatewayRoute, in any namespace, that references this backend with a matching + ModelNameOverride picks up these hints without repeating them. + + Since an AIServiceBackend has exactly one Schema, hints declared here are inherently scoped + to that provider — the same logical model hosted on a different AIServiceBackend (e.g. the + same Claude model on AWS Bedrock vs. GCP Vertex AI) is configured as a separate entry on + that other backend, which is correct because providers can genuinely support different + features for what is nominally the same model. + items: + description: |- + ModelTranslationHintsEntry associates ModelTranslationHints with a specific model routed + through this backend. + properties: + maxOutputTokens: + description: |- + MaxOutputTokens is the maximum number of tokens the model can produce in a single response. + When set, translators use this as the default max_tokens value for APIs that require it + (e.g., the Anthropic Messages API) when the client request omits max_tokens. + format: int64 + minimum: 1 + type: integer + modelNameOverride: + description: |- + ModelNameOverride identifies which model these hints apply to. Must match the + ModelNameOverride configured on an AIGatewayRouteRuleBackendRef that targets this backend + for the hints to take effect. + minLength: 1 + type: string + supportsOutputConfig: + description: |- + SupportsOutputConfig indicates whether the model accepts an Anthropic-style output_config + field for structured output (e.g., Claude Opus/Sonnet 4.5 and later). + type: boolean + supportsReasoningEffort: + description: |- + SupportsReasoningEffort indicates whether the model accepts a reasoning effort parameter + (Gemini thinking level / Anthropic output_config.effort). + type: boolean + supportsResponseJsonSchema: + description: |- + SupportsResponseJSONSchema indicates whether the model accepts a JSON Schema object in + response_format (as opposed to only {"type": "json_object"}). + type: boolean + required: + - modelNameOverride + type: object + type: array + x-kubernetes-list-map-keys: + - modelNameOverride x-kubernetes-list-type: map schema: description: |- diff --git a/site/docs/api/api.mdx b/site/docs/api/api.mdx index b559cebd54..4ef9c0bc2b 100644 --- a/site/docs/api/api.mdx +++ b/site/docs/api/api.mdx @@ -611,6 +611,7 @@ QuotaPolicyList contains a list of QuotaPolicy - [MCPRouteSpec](#github-com-envoyproxy-ai-gateway-api-v1alpha1-mcproutespec) - [MCPRouteStatus](#github-com-envoyproxy-ai-gateway-api-v1alpha1-mcproutestatus) - [MCPToolFilter](#github-com-envoyproxy-ai-gateway-api-v1alpha1-mcptoolfilter) +- [ModelTranslationHints](#github-com-envoyproxy-ai-gateway-api-v1alpha1-modeltranslationhints) - [PerModelQuota](#github-com-envoyproxy-ai-gateway-api-v1alpha1-permodelquota) - [ProtectedResourceMetadata](#github-com-envoyproxy-ai-gateway-api-v1alpha1-protectedresourcemetadata) - [QuotaBucketMode](#github-com-envoyproxy-ai-gateway-api-v1alpha1-quotabucketmode) @@ -725,6 +726,11 @@ It can reference either an AIServiceBackend or an InferencePool resource. type="[HTTPBodyMutation](#github-com-envoyproxy-ai-gateway-api-v1alpha1-httpbodymutation)" required="false" description="BodyMutation defines the request body mutation to be applied to this backend.
This allows modification of JSON fields in the request body before sending to the backend.
When both route-level and backend-level BodyMutation are defined,
route-level takes precedence over backend-level for conflicting operations.
This field is ignored when referencing InferencePool resources." +/> +#### ModelTranslationHints + + + +**Appears in:** +- [AIGatewayRouteRuleBackendRef](#github-com-envoyproxy-ai-gateway-api-v1alpha1-aigatewayrouterulebackendref) + +ModelTranslationHints provides model-specific hints to guide request and response translation +for a model-backend pairing. When set, translators use these values instead of model-name +heuristics. All fields are optional; a nil field falls back to the built-in heuristic for +backward compatibility. + +These hints are typically populated by a control plane from a model registry or catalog that +tracks per-model metadata such as maximum output token limits and supported features. + +##### Fields + + + + + + #### PerModelQuota @@ -3189,6 +3237,7 @@ MCPRouteList contains a list of MCPRoute. - [MCPRouteSpec](#github-com-envoyproxy-ai-gateway-api-v1beta1-mcproutespec) - [MCPRouteStatus](#github-com-envoyproxy-ai-gateway-api-v1beta1-mcproutestatus) - [MCPToolFilter](#github-com-envoyproxy-ai-gateway-api-v1beta1-mcptoolfilter) +- [ModelTranslationHints](#github-com-envoyproxy-ai-gateway-api-v1beta1-modeltranslationhints) - [ProtectedResourceMetadata](#github-com-envoyproxy-ai-gateway-api-v1beta1-protectedresourcemetadata) - [ToolCall](#github-com-envoyproxy-ai-gateway-api-v1beta1-toolcall) - [VersionedAPISchema](#github-com-envoyproxy-ai-gateway-api-v1beta1-versionedapischema) @@ -3295,6 +3344,11 @@ It can reference either an AIServiceBackend or an InferencePool resource. type="[HTTPBodyMutation](#github-com-envoyproxy-ai-gateway-api-v1beta1-httpbodymutation)" required="false" description="BodyMutation defines the request body mutation to be applied to this backend.
This allows modification of JSON fields in the request body before sending to the backend.
When both route-level and backend-level BodyMutation are defined,
route-level takes precedence over backend-level for conflicting operations.
This field is ignored when referencing InferencePool resources." +/> +#### ModelTranslationHints + + + +**Appears in:** +- [AIGatewayRouteRuleBackendRef](#github-com-envoyproxy-ai-gateway-api-v1beta1-aigatewayrouterulebackendref) + +ModelTranslationHints provides model-specific hints to guide request and response translation +for a model-backend pairing. When set, translators use these values instead of model-name +heuristics. All fields are optional; a nil field falls back to the built-in heuristic for +backward compatibility. + +These hints are typically populated by a control plane from a model registry or catalog that +tracks per-model metadata such as maximum output token limits and supported features. + +##### Fields + + + + + + #### ProtectedResourceMetadata From ffbb146b872a7ebf6fc309897088413f171531ea Mon Sep 17 00:00:00 2001 From: yxia216 Date: Tue, 22 Sep 2026 10:13:32 -0400 Subject: [PATCH 2/3] fix: update endpoint translator signatures Signed-off-by: yxia216 --- internal/endpointspec/endpointspec.go | 20 +++++++++++++------- internal/endpointspec/endpointspec_test.go | 18 +++++++++--------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/internal/endpointspec/endpointspec.go b/internal/endpointspec/endpointspec.go index 64dffcda8f..cff906097f 100644 --- a/internal/endpointspec/endpointspec.go +++ b/internal/endpointspec/endpointspec.go @@ -471,7 +471,9 @@ func (MessagesCountTokensEndpointSpec) ParseBody( } // GetTranslator implements [EndpointSpec.GetTranslator]. -func (MessagesCountTokensEndpointSpec) GetTranslator(schema filterapi.VersionedAPISchema, modelNameOverride string) (translator.AnthropicCountTokensTranslator, error) { +func (MessagesCountTokensEndpointSpec) GetTranslator(backend *filterapi.Backend) (translator.AnthropicCountTokensTranslator, error) { + schema := backend.Schema + modelNameOverride := backend.ModelNameOverride switch schema.Name { case filterapi.APISchemaGCPAnthropic: return translator.NewCountTokensToGCPAnthropicTranslator(schema.Version, modelNameOverride), nil @@ -552,10 +554,11 @@ func (SystemOneEndpointSpec) ParseMultipartBody([]byte, string, bool) (internala } // GetTranslator implements [EndpointSpec.GetTranslator]. -func (SystemOneEndpointSpec) GetTranslator(schema filterapi.VersionedAPISchema, modelNameOverride string) (translator.TypeSafeSystemOneTranslator, error) { +func (SystemOneEndpointSpec) GetTranslator(backend *filterapi.Backend) (translator.TypeSafeSystemOneTranslator, error) { + schema := backend.Schema switch schema.Name { case filterapi.APISchemaTypeSafe: - return translator.NewSystemOneTypeSafeToTypeSafeTranslator(schema.Version, modelNameOverride), nil + return translator.NewSystemOneTypeSafeToTypeSafeTranslator(schema.Version, backend.ModelNameOverride), nil default: return nil, fmt.Errorf("unsupported API schema: backend=%s", schema) } @@ -618,7 +621,9 @@ func (TokenizeEndpointSpec) ParseBody( } // GetTranslator implements [EndpointSpec.GetTranslator]. -func (TokenizeEndpointSpec) GetTranslator(schema filterapi.VersionedAPISchema, modelNameOverride string) (translator.TokenizeTranslator, error) { +func (TokenizeEndpointSpec) GetTranslator(backend *filterapi.Backend) (translator.TokenizeTranslator, error) { + schema := backend.Schema + modelNameOverride := backend.ModelNameOverride switch schema.Name { case filterapi.APISchemaOpenAI: return translator.NewTokenizeTranslator(modelNameOverride), nil @@ -1186,13 +1191,14 @@ func (ResponsesInputTokensEndpointSpec) ParseBody( // GetTranslator implements [EndpointSpec.GetTranslator]. func (ResponsesInputTokensEndpointSpec) GetTranslator( - schema filterapi.VersionedAPISchema, modelNameOverride string, + backend *filterapi.Backend, ) (translator.OpenAIResponsesInputTokensTranslator, error) { + schema := backend.Schema switch schema.Name { case filterapi.APISchemaOpenAI: - return translator.NewResponsesInputTokensOpenAIToOpenAITranslator(schema.OpenAIPrefix(), modelNameOverride), nil + return translator.NewResponsesInputTokensOpenAIToOpenAITranslator(schema.OpenAIPrefix(), backend.ModelNameOverride), nil case filterapi.APISchemaAzureOpenAI: - return translator.NewResponsesInputTokensOpenAIToAzureOpenAITranslator(schema.Version, modelNameOverride), nil + return translator.NewResponsesInputTokensOpenAIToAzureOpenAITranslator(schema.Version, backend.ModelNameOverride), nil default: return nil, fmt.Errorf("unsupported API schema for /v1/responses/input_tokens: backend=%s", schema) } diff --git a/internal/endpointspec/endpointspec_test.go b/internal/endpointspec/endpointspec_test.go index a7f8aa29e0..0da71f14ef 100644 --- a/internal/endpointspec/endpointspec_test.go +++ b/internal/endpointspec/endpointspec_test.go @@ -352,10 +352,10 @@ func TestSystemOneEndpointSpec_ParseBody(t *testing.T) { func TestSystemOneEndpointSpec_GetTranslator(t *testing.T) { spec := SystemOneEndpointSpec{} - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaTypeSafe, Version: "v1"}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaTypeSafe, Version: "v1"}, ModelNameOverride: "override"}) require.NoError(t, err) - _, err = spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, "override") + _, err = spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "unsupported API schema") } @@ -553,14 +553,14 @@ func TestTokenizeEndpointSpec_GetTranslator(t *testing.T) { s := schema t.Run("supported_"+string(s.Name), func(t *testing.T) { t.Parallel() - translator, err := spec.GetTranslator(s, "override") + translator, err := spec.GetTranslator(&filterapi.Backend{Schema: s, ModelNameOverride: "override"}) require.NoError(t, err) require.NotNil(t, translator) }) } t.Run("unsupported", func(t *testing.T) { - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: "Unknown"}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: "Unknown"}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "unsupported API schema for tokenize endpoint") }) } @@ -1583,13 +1583,13 @@ func TestResponsesInputTokensEndpointSpec_ParseBody(t *testing.T) { func TestResponsesInputTokensEndpointSpec_GetTranslator(t *testing.T) { spec := ResponsesInputTokensEndpointSpec{} - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, ModelNameOverride: "override"}) require.NoError(t, err) - _, err = spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaAzureOpenAI, Version: "2025-01-01-preview"}, "override") + _, err = spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAzureOpenAI, Version: "2025-01-01-preview"}, ModelNameOverride: "override"}) require.NoError(t, err) - _, err = spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaAnthropic}, "override") + _, err = spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaAnthropic}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "unsupported API schema") } @@ -1850,11 +1850,11 @@ func TestMessagesCountTokensEndpointSpec_GetTranslator(t *testing.T) { {Name: filterapi.APISchemaAWSAnthropic}, {Name: filterapi.APISchemaAnthropic}, } { - translator, err := spec.GetTranslator(schema, "override") + translator, err := spec.GetTranslator(&filterapi.Backend{Schema: schema, ModelNameOverride: "override"}) require.NoError(t, err) require.NotNil(t, translator) } - _, err := spec.GetTranslator(filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, "override") + _, err := spec.GetTranslator(&filterapi.Backend{Schema: filterapi.VersionedAPISchema{Name: filterapi.APISchemaOpenAI}, ModelNameOverride: "override"}) require.ErrorContains(t, err, "unsupported") } From f5bda057eb303ec44f79bbb17e543fc5e8421dee Mon Sep 17 00:00:00 2001 From: yxia216 Date: Tue, 22 Sep 2026 13:00:10 -0400 Subject: [PATCH 3/3] chore: regenerate CRDs and API documentation Signed-off-by: yxia216 --- ...teway.envoyproxy.io_aiservicebackends.yaml | 1 + site/docs/api/api.mdx | 124 +++++++++++++++--- 2 files changed, 105 insertions(+), 20 deletions(-) diff --git a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_aiservicebackends.yaml b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_aiservicebackends.yaml index f4baf0af0a..c8d9e2a5f8 100644 --- a/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_aiservicebackends.yaml +++ b/manifests/charts/ai-gateway-crds-helm/templates/aigateway.envoyproxy.io_aiservicebackends.yaml @@ -834,6 +834,7 @@ spec: type: array x-kubernetes-list-map-keys: - name + x-kubernetes-list-type: map modelTranslationHints: description: |- ModelTranslationHints declares translation hints for models routed through this backend, diff --git a/site/docs/api/api.mdx b/site/docs/api/api.mdx index 4ef9c0bc2b..948e740f10 100644 --- a/site/docs/api/api.mdx +++ b/site/docs/api/api.mdx @@ -612,6 +612,7 @@ QuotaPolicyList contains a list of QuotaPolicy - [MCPRouteStatus](#github-com-envoyproxy-ai-gateway-api-v1alpha1-mcproutestatus) - [MCPToolFilter](#github-com-envoyproxy-ai-gateway-api-v1alpha1-mcptoolfilter) - [ModelTranslationHints](#github-com-envoyproxy-ai-gateway-api-v1alpha1-modeltranslationhints) +- [ModelTranslationHintsEntry](#github-com-envoyproxy-ai-gateway-api-v1alpha1-modeltranslationhintsentry) - [PerModelQuota](#github-com-envoyproxy-ai-gateway-api-v1alpha1-permodelquota) - [ProtectedResourceMetadata](#github-com-envoyproxy-ai-gateway-api-v1alpha1-protectedresourcemetadata) - [QuotaBucketMode](#github-com-envoyproxy-ai-gateway-api-v1alpha1-quotabucketmode) @@ -726,11 +727,6 @@ It can reference either an AIServiceBackend or an InferencePool resource. type="[HTTPBodyMutation](#github-com-envoyproxy-ai-gateway-api-v1alpha1-httpbodymutation)" required="false" description="BodyMutation defines the request body mutation to be applied to this backend.
This allows modification of JSON fields in the request body before sending to the backend.
When both route-level and backend-level BodyMutation are defined,
route-level takes precedence over backend-level for conflicting operations.
This field is ignored when referencing InferencePool resources." -/> @@ -2341,12 +2342,11 @@ a tool must match an include rule AND not match any exclude rule to be allowed. **Appears in:** -- [AIGatewayRouteRuleBackendRef](#github-com-envoyproxy-ai-gateway-api-v1alpha1-aigatewayrouterulebackendref) +- [ModelTranslationHintsEntry](#github-com-envoyproxy-ai-gateway-api-v1alpha1-modeltranslationhintsentry) -ModelTranslationHints provides model-specific hints to guide request and response translation -for a model-backend pairing. When set, translators use these values instead of model-name -heuristics. All fields are optional; a nil field falls back to the built-in heuristic for -backward compatibility. +ModelTranslationHints provides model-specific hints to guide request and response translation. +When set, translators use these values instead of model-name heuristics. All fields are +optional; a nil field falls back to the built-in heuristic for backward compatibility. These hints are typically populated by a control plane from a model registry or catalog that tracks per-model metadata such as maximum output token limits and supported features. @@ -2378,6 +2378,48 @@ tracks per-model metadata such as maximum output token limits and supported feat /> +#### ModelTranslationHintsEntry + + + +**Appears in:** +- [AIServiceBackendSpec](#github-com-envoyproxy-ai-gateway-api-v1alpha1-aiservicebackendspec) + +ModelTranslationHintsEntry associates ModelTranslationHints with a specific model routed +through this backend. + +##### Fields + + + + + + #### PerModelQuota @@ -3238,6 +3280,7 @@ MCPRouteList contains a list of MCPRoute. - [MCPRouteStatus](#github-com-envoyproxy-ai-gateway-api-v1beta1-mcproutestatus) - [MCPToolFilter](#github-com-envoyproxy-ai-gateway-api-v1beta1-mcptoolfilter) - [ModelTranslationHints](#github-com-envoyproxy-ai-gateway-api-v1beta1-modeltranslationhints) +- [ModelTranslationHintsEntry](#github-com-envoyproxy-ai-gateway-api-v1beta1-modeltranslationhintsentry) - [ProtectedResourceMetadata](#github-com-envoyproxy-ai-gateway-api-v1beta1-protectedresourcemetadata) - [ToolCall](#github-com-envoyproxy-ai-gateway-api-v1beta1-toolcall) - [VersionedAPISchema](#github-com-envoyproxy-ai-gateway-api-v1beta1-versionedapischema) @@ -3344,11 +3387,6 @@ It can reference either an AIServiceBackend or an InferencePool resource. type="[HTTPBodyMutation](#github-com-envoyproxy-ai-gateway-api-v1beta1-httpbodymutation)" required="false" description="BodyMutation defines the request body mutation to be applied to this backend.
This allows modification of JSON fields in the request body before sending to the backend.
When both route-level and backend-level BodyMutation are defined,
route-level takes precedence over backend-level for conflicting operations.
This field is ignored when referencing InferencePool resources." -/> @@ -5247,12 +5290,11 @@ a tool must match an include rule AND not match any exclude rule to be allowed. **Appears in:** -- [AIGatewayRouteRuleBackendRef](#github-com-envoyproxy-ai-gateway-api-v1beta1-aigatewayrouterulebackendref) +- [ModelTranslationHintsEntry](#github-com-envoyproxy-ai-gateway-api-v1beta1-modeltranslationhintsentry) -ModelTranslationHints provides model-specific hints to guide request and response translation -for a model-backend pairing. When set, translators use these values instead of model-name -heuristics. All fields are optional; a nil field falls back to the built-in heuristic for -backward compatibility. +ModelTranslationHints provides model-specific hints to guide request and response translation. +When set, translators use these values instead of model-name heuristics. All fields are +optional; a nil field falls back to the built-in heuristic for backward compatibility. These hints are typically populated by a control plane from a model registry or catalog that tracks per-model metadata such as maximum output token limits and supported features. @@ -5284,6 +5326,48 @@ tracks per-model metadata such as maximum output token limits and supported feat /> +#### ModelTranslationHintsEntry + + + +**Appears in:** +- [AIServiceBackendSpec](#github-com-envoyproxy-ai-gateway-api-v1beta1-aiservicebackendspec) + +ModelTranslationHintsEntry associates ModelTranslationHints with a specific model routed +through this backend. + +##### Fields + + + + + + #### ProtectedResourceMetadata