From 9d3c319796fe74d502d5f53ee5942620699f60d4 Mon Sep 17 00:00:00 2001 From: Connor Carpenter Date: Wed, 15 Jul 2026 09:37:15 -0700 Subject: [PATCH 1/2] refactor(api): separate deployment and routing metadata Signed-off-by: Connor Carpenter --- README.md | 4 +- docs/api.md | 90 ++++++++++++++++++++++------ proto/openengine/v1/README.md | 2 +- proto/openengine/v1/generation.proto | 13 ++-- proto/openengine/v1/model.proto | 9 +-- proto/openengine/v1/openengine.proto | 12 ++++ proto/openengine/v1/server.proto | 17 +++++- proto/openengine/v1/tasks.proto | 7 ++- 8 files changed, 117 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 13e5d30..4dcd880 100644 --- a/README.md +++ b/README.md @@ -105,11 +105,11 @@ The canonical schema is organized by domain under | Area | What the contract provides | | --------------------- | ----------------------------------------------------------------------------------------------------------------- | -| Portable generation | Text or token input, sampling, stopping, priorities, multiple sequences, and deterministic seeds | +| Portable generation | Text or token input, sampling, stopping, transport priorities, multiple sequences, and deterministic seeds | | Non-generative tasks | Typed embedding, classification, and grouped query/candidate scoring with stable correlation | | Structured output | JSON Schema, JSON object, regex, EBNF grammar, structural tags, and fixed choices | | Token information | Prompt and output logprobs, ranks, candidate-token selection, per-token records, and streamed text deltas | -| Discovery | Server identity, schema revision, role, model limits, topology, parser configuration, and generation capabilities | +| Discovery | Server identity, deployment capacity, model limits, topology, parsers, and inference capabilities | | Lifecycle | Health checks, targeted or global abort, graceful drain, progress, and terminal failures | | Disaggregated serving | Prefill/decode roles, decode-context parallel topology, KV handoff, connector discovery, and cache controls | | KV-aware routing | Typed KV event streams plus discovery of engine-native event sources | diff --git a/docs/api.md b/docs/api.md index 98d2a6b..0b75134 100644 --- a/docs/api.md +++ b/docs/api.md @@ -61,9 +61,38 @@ and coordination control plane. Implementations may expose both services on the same listener, or isolate them on separate listeners and access policies without changing the protocol contract. +### gRPC request metadata + +Routing, admission, and tracing context belongs in +[gRPC request metadata](https://grpc.io/docs/guides/metadata/) rather than +protobuf request payloads. OpenEngine defines these lowercase ASCII keys: + +| Key | Value | Scope | +| --- | --- | --- | +| `openengine-routing-key` | Opaque non-empty ASCII routing key | Consistent hashing, tenancy, or other application routing | +| `openengine-target-dp-rank` | Base-10 `uint32` | Requested data-parallel routing target | +| `openengine-priority` | Base-10 `int32` | Admission priority; higher values have higher priority | +| `traceparent` | W3C Trace Context value | Portable distributed-trace parent | +| `tracestate` | W3C Trace Context value | Vendor-specific trace state associated with `traceparent` | + +The `openengine-` prefix is reserved for this protocol. Clients send at most one +value for each OpenEngine key; malformed numeric values or repeated OpenEngine +keys return gRPC `INVALID_ARGUMENT`. Application-specific metadata may use other +gRPC metadata keys without adding fields to inference messages. + +`openengine-target-dp-rank` is a routing instruction, not KV state. When a +request carries a `KvSessionRef`, its `dp_rank` is the authoritative KV-affinity +value. If both values reach the engine and disagree, the engine returns +`INVALID_ARGUMENT`. The priority key is used only when the selected model and +task advertise priority support. + +All RPCs accept `traceparent` and `tracestate` and propagate them on downstream +RPCs according to [W3C Trace Context](https://www.w3.org/TR/trace-context/). +Routing and admission keys apply only to `Inference` RPCs. + --- -## Server identity and engine roles +## Server identity, deployment capacity, and engine roles ```protobuf enum EngineRole { @@ -86,6 +115,15 @@ message ServerInfo { uint32 schema_revision = 8; uint32 minimum_client_revision = 9; string schema_release = 10; + DeploymentCapacity capacity = 11; +} + +message DeploymentCapacity { + optional uint32 kv_block_size = 1; + optional uint64 total_kv_blocks = 2; + optional uint64 max_running_requests = 3; + optional uint64 max_batched_tokens = 4; + optional uint32 max_loras = 5; } message ParallelismInfo { @@ -126,6 +164,13 @@ and does not, by itself, imply additional workers beyond the reported tensor, pipeline, and data-parallel topology. When present, the value must be at least one; one means decode-context parallelism is disabled. +`DeploymentCapacity` reports configured capacity for the deployed server, not +model identity. `kv_block_size` is the number of tokens in a deployed KV block; +`total_kv_blocks` is the allocatable block count in the server's reporting +scope; `max_running_requests` and `max_batched_tokens` are configured scheduler +ceilings; and `max_loras` is the maximum number of simultaneously resident LoRA +adapters. Dynamic and per-rank utilization remains in `LoadInfo`. + Role semantics: - `AGGREGATED`: accepts normal generation requests and returns tokens. @@ -136,7 +181,7 @@ Role semantics: --- -## Model and capacity metadata +## Model identity and capabilities ```protobuf message GetModelInfoRequest { @@ -144,15 +189,14 @@ message GetModelInfoRequest { } message ModelInfo { + reserved 6 to 9; + reserved "kv_block_size", "total_kv_blocks", "max_running_requests", "max_batched_tokens"; + string model_id = 1; string served_model_name = 2; repeated string served_model_aliases = 3; optional uint32 max_context_length = 4; optional uint32 max_output_tokens = 5; - optional uint32 kv_block_size = 6; - optional uint64 total_kv_blocks = 7; - optional uint64 max_running_requests = 8; - optional uint64 max_batched_tokens = 9; repeated string tokenizer_modes = 10; optional bool supports_text_input = 20; @@ -208,11 +252,15 @@ enum GuidedDecodingMode { `GetModelInfoRequest.model` is required and selects one of `ServerInfo.supported_models`; an unknown model returns gRPC `NOT_FOUND`. +`max_context_length` and `max_output_tokens` are the effective limits for the +selected model in this deployment. KV layout and scheduler capacity are reported +once through `ServerInfo.capacity`, not repeated as model identity. Capability submessages distinguish unreported support (message absent) from reported support or lack of support (`supported = true` or `false`). Candidate selection modes and `max_top_n` are reported independently for prompt and -output logprobs. The remaining generation fields advertise support and limits -for the corresponding request options. +output logprobs. `supports_priority` advertises support for the +`openengine-priority` metadata key. The remaining generation fields advertise +support and limits for the corresponding request options. `supports_lora=true` means the engine accepts `GenerateRequest.lora_name` and the LoRA lifecycle RPCs on `Control`. @@ -227,11 +275,12 @@ Support is optional per model and advertised through `ModelInfo.tasks`. ```protobuf message TaskRequestContext { + reserved 4, 5; + reserved "priority", "metadata"; + string request_id = 1; string model = 2; string lora_name = 3; - optional int32 priority = 4; - map metadata = 5; } message TaskInput { @@ -306,10 +355,9 @@ enum ScoreNormalization { `TaskRequestContext.request_id` and `model` are required and non-empty. Request IDs share the same namespace and abort semantics as generation request IDs. -`priority` uses the generation ordering convention: larger values have higher -priority. A non-empty `lora_name` selects an already loaded adapter. Clients -must use either option only when the corresponding task capability advertises -support. +A non-empty `lora_name` selects an already loaded adapter. Clients use +`openengine-priority` only when the corresponding task capability advertises +priority support, and use `lora_name` only when it advertises LoRA support. Each request batch must be non-empty. `item_id` is required and unique within an embed/classify batch, and every query/candidate item ID is unique within one @@ -615,6 +663,9 @@ Generation is the core runtime completion primitive. Frontends or gateways may l ```protobuf message GenerateRequest { + reserved 12, 13; + reserved "priority", "metadata"; + string request_id = 1; string model = 2; @@ -631,8 +682,6 @@ message GenerateRequest { repeated MediaItem media = 10; string lora_name = 11; - optional int32 priority = 12; - map metadata = 13; } message TokenIds { @@ -678,8 +727,10 @@ message CandidateTokenSelection { message AllCandidates {} message KvOptions { + reserved 2; + reserved "data_parallel_rank"; + KvSessionRef session = 1; - optional uint32 data_parallel_rank = 2; optional bool bypass_prefix_cache = 3; optional string cache_salt = 4; } @@ -738,8 +789,9 @@ data, and KV/cache behavior remain separate option groups. Guided decoding stays top-level as a distinct structured-output mode. Optional scalars preserve the distinction between an engine default and explicit zero or `false`. -`priority` uses higher values for higher scheduling priority. `num_sequences` -defaults to one when omitted and must be greater than zero when present. +`openengine-priority` uses higher values for higher scheduling priority. +`num_sequences` defaults to one when omitted and must be greater than zero when +present. `CandidateTokenSelection` requests either the top N candidates, explicit token IDs, or the full vocabulary at each prompt or output position. Select all candidates with `all {}` and JSON-object guidance with `json_object {}`. diff --git a/proto/openengine/v1/README.md b/proto/openengine/v1/README.md index 49b85a5..f9f8a0c 100644 --- a/proto/openengine/v1/README.md +++ b/proto/openengine/v1/README.md @@ -12,7 +12,7 @@ share the same package and together define the API. | --- | --- | | [`openengine.proto`](openengine.proto) | `Inference` and `Control` service declarations | | [`input.proto`](input.proto) | Shared text, token, and multimodal inputs | -| [`server.proto`](server.proto) | Server identity, engine roles, and parallelism | +| [`server.proto`](server.proto) | Server identity, deployment capacity, engine roles, and parallelism | | [`model.proto`](model.proto) | Model metadata and inference capabilities | | [`generation.proto`](generation.proto) | Generation requests, parameters, streamed events, and usage | | [`tasks.proto`](tasks.proto) | Shared non-generative task request and output vocabulary | diff --git a/proto/openengine/v1/generation.proto b/proto/openengine/v1/generation.proto index 34ed557..364ff61 100644 --- a/proto/openengine/v1/generation.proto +++ b/proto/openengine/v1/generation.proto @@ -12,6 +12,9 @@ import "openengine/v1/input.proto"; import "openengine/v1/kv.proto"; message GenerateRequest { + reserved 12, 13; + reserved "priority", "metadata"; + string request_id = 1; string model = 2; @@ -36,12 +39,6 @@ message GenerateRequest { // ModelInfo.supports_lora advertises whether lifecycle and selection are // available through OpenEngine. string lora_name = 11; - - // Higher values receive higher scheduling priority. - optional int32 priority = 12; - - // Optional request metadata for tracing/admission/routing. - map metadata = 13; } message SamplingParams { @@ -83,8 +80,10 @@ message CandidateTokenSelection { message AllCandidates {} message KvOptions { + reserved 2; + reserved "data_parallel_rank"; + KvSessionRef session = 1; - optional uint32 data_parallel_rank = 2; optional bool bypass_prefix_cache = 3; optional string cache_salt = 4; } diff --git a/proto/openengine/v1/model.proto b/proto/openengine/v1/model.proto index 3cfb72c..d09e5c1 100644 --- a/proto/openengine/v1/model.proto +++ b/proto/openengine/v1/model.proto @@ -16,15 +16,16 @@ message GetModelInfoRequest { } message ModelInfo { + reserved 6 to 9; + reserved "kv_block_size", "total_kv_blocks", "max_running_requests", "max_batched_tokens"; + string model_id = 1; string served_model_name = 2; repeated string served_model_aliases = 3; + // Effective context-window limit for this model in the current deployment. optional uint32 max_context_length = 4; + // Effective generated-token limit for this model in the current deployment. optional uint32 max_output_tokens = 5; - optional uint32 kv_block_size = 6; - optional uint64 total_kv_blocks = 7; - optional uint64 max_running_requests = 8; - optional uint64 max_batched_tokens = 9; repeated string tokenizer_modes = 10; optional bool supports_text_input = 20; diff --git a/proto/openengine/v1/openengine.proto b/proto/openengine/v1/openengine.proto index 8526814..8a50f7c 100644 --- a/proto/openengine/v1/openengine.proto +++ b/proto/openengine/v1/openengine.proto @@ -27,6 +27,18 @@ import "openengine/v1/observability.proto"; import "openengine/v1/scoring.proto"; import "openengine/v1/server.proto"; +// Standard ASCII gRPC request metadata: +// - openengine-routing-key: opaque application routing key. +// - openengine-target-dp-rank: base-10 uint32 routing target. This does not +// establish KV affinity; KvSessionRef.dp_rank is authoritative for a session. +// - openengine-priority: base-10 int32 admission priority; higher values have +// higher priority. +// - traceparent and tracestate: W3C Trace Context. +// +// The openengine- prefix is reserved for metadata defined by this protocol. +// Routing and admission keys apply to Inference RPCs. Trace Context applies to +// every RPC and should be propagated across downstream calls. + service Inference { // Core inference path. rpc Generate(GenerateRequest) returns (stream GenerateResponse); diff --git a/proto/openengine/v1/server.proto b/proto/openengine/v1/server.proto index dd247a9..19c5724 100644 --- a/proto/openengine/v1/server.proto +++ b/proto/openengine/v1/server.proto @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -// Server identity, engine role, and parallelism metadata. +// Server identity, deployment capacity, engine role, and parallelism metadata. syntax = "proto3"; @@ -29,6 +29,21 @@ message ServerInfo { uint32 schema_revision = 8; // Monotonic wire contract revision; zero is invalid. uint32 minimum_client_revision = 9; // Oldest compatible client revision. string schema_release = 10; // Immutable release or source tag for this schema. + // Configured capacity for this deployed server, not for an abstract model. + DeploymentCapacity capacity = 11; +} + +message DeploymentCapacity { + // Tokens represented by one block in the deployed KV-cache layout. + optional uint32 kv_block_size = 1; + // Allocatable KV blocks in the server's capacity-reporting scope. + optional uint64 total_kv_blocks = 2; + // Configured ceiling for concurrently running requests. + optional uint64 max_running_requests = 3; + // Configured scheduler ceiling for tokens in one batch. + optional uint64 max_batched_tokens = 4; + // Maximum number of LoRA adapters that may be resident simultaneously. + optional uint32 max_loras = 5; } message ParallelismInfo { diff --git a/proto/openengine/v1/tasks.proto b/proto/openengine/v1/tasks.proto index 2338885..b1fea1f 100644 --- a/proto/openengine/v1/tasks.proto +++ b/proto/openengine/v1/tasks.proto @@ -7,13 +7,14 @@ syntax = "proto3"; package openengine.v1; -// Request metadata shared by every non-generative task. +// Request identity and model selection shared by every non-generative task. message TaskRequestContext { + reserved 4, 5; + reserved "priority", "metadata"; + string request_id = 1; string model = 2; string lora_name = 3; - optional int32 priority = 4; - map metadata = 5; } // Row-major FP32 tensor. The product of shape must equal values.size(). From 523a3b2bd53c87c5f88342984499623f34411f5c Mon Sep 17 00:00:00 2001 From: Connor Carpenter Date: Wed, 15 Jul 2026 09:46:03 -0700 Subject: [PATCH 2/2] refactor(api): remove field reservations Signed-off-by: Connor Carpenter --- docs/api.md | 33 +++++++++------------------- proto/openengine/v1/generation.proto | 6 ----- proto/openengine/v1/model.proto | 12 +++------- proto/openengine/v1/server.proto | 21 ++++++------------ proto/openengine/v1/tasks.proto | 3 --- 5 files changed, 20 insertions(+), 55 deletions(-) diff --git a/docs/api.md b/docs/api.md index 0b75134..c8b24a8 100644 --- a/docs/api.md +++ b/docs/api.md @@ -115,15 +115,15 @@ message ServerInfo { uint32 schema_revision = 8; uint32 minimum_client_revision = 9; string schema_release = 10; - DeploymentCapacity capacity = 11; + DeploymentCapacity capacity = 11; // Configured capacity for this deployed server. } message DeploymentCapacity { - optional uint32 kv_block_size = 1; - optional uint64 total_kv_blocks = 2; - optional uint64 max_running_requests = 3; - optional uint64 max_batched_tokens = 4; - optional uint32 max_loras = 5; + optional uint32 kv_block_size = 1; // Tokens per deployed KV-cache block. + optional uint64 total_kv_blocks = 2; // Allocatable KV blocks in the reporting scope. + optional uint64 max_running_requests = 3; // Concurrent running-request ceiling. + optional uint64 max_batched_tokens = 4; // Scheduler token ceiling per batch. + optional uint32 max_loras = 5; // Maximum simultaneously resident LoRA adapters. } message ParallelismInfo { @@ -132,8 +132,7 @@ message ParallelismInfo { optional uint32 data_parallel_size = 3; optional uint32 data_parallel_rank = 4; optional uint32 data_parallel_start_rank = 5; - // Number of ranks in each decode-context-parallel group. Must be at least 1. - optional uint32 decode_context_parallel_size = 6; + optional uint32 decode_context_parallel_size = 6; // Ranks per decode-context group; at least 1. } ``` @@ -189,14 +188,11 @@ message GetModelInfoRequest { } message ModelInfo { - reserved 6 to 9; - reserved "kv_block_size", "total_kv_blocks", "max_running_requests", "max_batched_tokens"; - string model_id = 1; string served_model_name = 2; repeated string served_model_aliases = 3; - optional uint32 max_context_length = 4; - optional uint32 max_output_tokens = 5; + optional uint32 max_context_length = 4; // Effective context-window limit in this deployment. + optional uint32 max_output_tokens = 5; // Effective generated-token limit in this deployment. repeated string tokenizer_modes = 10; optional bool supports_text_input = 20; @@ -207,7 +203,7 @@ message ModelInfo { string reasoning_parser = 25; string tool_call_parser = 26; - TaskCapabilities tasks = 27; + TaskCapabilities tasks = 27; // Optional non-generative task support for this model. } message GenerationCapabilities { @@ -275,9 +271,6 @@ Support is optional per model and advertised through `ModelInfo.tasks`. ```protobuf message TaskRequestContext { - reserved 4, 5; - reserved "priority", "metadata"; - string request_id = 1; string model = 2; string lora_name = 3; @@ -663,9 +656,6 @@ Generation is the core runtime completion primitive. Frontends or gateways may l ```protobuf message GenerateRequest { - reserved 12, 13; - reserved "priority", "metadata"; - string request_id = 1; string model = 2; @@ -727,9 +717,6 @@ message CandidateTokenSelection { message AllCandidates {} message KvOptions { - reserved 2; - reserved "data_parallel_rank"; - KvSessionRef session = 1; optional bool bypass_prefix_cache = 3; optional string cache_salt = 4; diff --git a/proto/openengine/v1/generation.proto b/proto/openengine/v1/generation.proto index 364ff61..287697d 100644 --- a/proto/openengine/v1/generation.proto +++ b/proto/openengine/v1/generation.proto @@ -12,9 +12,6 @@ import "openengine/v1/input.proto"; import "openengine/v1/kv.proto"; message GenerateRequest { - reserved 12, 13; - reserved "priority", "metadata"; - string request_id = 1; string model = 2; @@ -80,9 +77,6 @@ message CandidateTokenSelection { message AllCandidates {} message KvOptions { - reserved 2; - reserved "data_parallel_rank"; - KvSessionRef session = 1; optional bool bypass_prefix_cache = 3; optional string cache_salt = 4; diff --git a/proto/openengine/v1/model.proto b/proto/openengine/v1/model.proto index d09e5c1..24a062c 100644 --- a/proto/openengine/v1/model.proto +++ b/proto/openengine/v1/model.proto @@ -16,16 +16,11 @@ message GetModelInfoRequest { } message ModelInfo { - reserved 6 to 9; - reserved "kv_block_size", "total_kv_blocks", "max_running_requests", "max_batched_tokens"; - string model_id = 1; string served_model_name = 2; repeated string served_model_aliases = 3; - // Effective context-window limit for this model in the current deployment. - optional uint32 max_context_length = 4; - // Effective generated-token limit for this model in the current deployment. - optional uint32 max_output_tokens = 5; + optional uint32 max_context_length = 4; // Effective context-window limit in this deployment. + optional uint32 max_output_tokens = 5; // Effective generated-token limit in this deployment. repeated string tokenizer_modes = 10; optional bool supports_text_input = 20; @@ -40,8 +35,7 @@ message ModelInfo { string reasoning_parser = 25; string tool_call_parser = 26; - // Optional non-generative task support for this model. - TaskCapabilities tasks = 27; + TaskCapabilities tasks = 27; // Optional non-generative task support for this model. } message TaskCapabilities { diff --git a/proto/openengine/v1/server.proto b/proto/openengine/v1/server.proto index 19c5724..81001a6 100644 --- a/proto/openengine/v1/server.proto +++ b/proto/openengine/v1/server.proto @@ -29,21 +29,15 @@ message ServerInfo { uint32 schema_revision = 8; // Monotonic wire contract revision; zero is invalid. uint32 minimum_client_revision = 9; // Oldest compatible client revision. string schema_release = 10; // Immutable release or source tag for this schema. - // Configured capacity for this deployed server, not for an abstract model. - DeploymentCapacity capacity = 11; + DeploymentCapacity capacity = 11; // Configured capacity for this deployed server. } message DeploymentCapacity { - // Tokens represented by one block in the deployed KV-cache layout. - optional uint32 kv_block_size = 1; - // Allocatable KV blocks in the server's capacity-reporting scope. - optional uint64 total_kv_blocks = 2; - // Configured ceiling for concurrently running requests. - optional uint64 max_running_requests = 3; - // Configured scheduler ceiling for tokens in one batch. - optional uint64 max_batched_tokens = 4; - // Maximum number of LoRA adapters that may be resident simultaneously. - optional uint32 max_loras = 5; + optional uint32 kv_block_size = 1; // Tokens per deployed KV-cache block. + optional uint64 total_kv_blocks = 2; // Allocatable KV blocks in the reporting scope. + optional uint64 max_running_requests = 3; // Concurrent running-request ceiling. + optional uint64 max_batched_tokens = 4; // Scheduler token ceiling per batch. + optional uint32 max_loras = 5; // Maximum simultaneously resident LoRA adapters. } message ParallelismInfo { @@ -52,6 +46,5 @@ message ParallelismInfo { optional uint32 data_parallel_size = 3; optional uint32 data_parallel_rank = 4; optional uint32 data_parallel_start_rank = 5; - // Number of ranks in each decode-context-parallel group. Must be at least 1. - optional uint32 decode_context_parallel_size = 6; + optional uint32 decode_context_parallel_size = 6; // Ranks per decode-context group; at least 1. } diff --git a/proto/openengine/v1/tasks.proto b/proto/openengine/v1/tasks.proto index b1fea1f..bc84006 100644 --- a/proto/openengine/v1/tasks.proto +++ b/proto/openengine/v1/tasks.proto @@ -9,9 +9,6 @@ package openengine.v1; // Request identity and model selection shared by every non-generative task. message TaskRequestContext { - reserved 4, 5; - reserved "priority", "metadata"; - string request_id = 1; string model = 2; string lora_name = 3;