diff --git a/README.md b/README.md index 5451c29..13e5d30 100644 --- a/README.md +++ b/README.md @@ -52,8 +52,9 @@ SPDX-License-Identifier: Apache-2.0 ## Overview OpenEngine defines a runtime boundary around an inference engine. An engine -exposes the `openengine.v1.OpenEngine` gRPC service, which applications can call -directly or distributed frameworks can use to coordinate engine workers. +exposes the `openengine.v1.Inference` and `openengine.v1.Control` services. +Applications can call the inference service directly, while distributed +frameworks use both services to coordinate engine workers. Both paths use generated clients and the same typed contract without sharing a process, Python environment, dependency tree, or private control API. @@ -67,7 +68,7 @@ scheduler implementation details. | Without a shared contract | With OpenEngine | | --------------------------------------------------- | --------------------------------------------------- | -| Engine-specific clients and framework integrations | One generated service contract | +| Engine-specific clients and framework integrations | One generated protocol contract | | Configuration duplicated into sidecars | Engine capabilities discovered over RPC | | Engine upgrades coupled to framework code | Engine-native execution behind a common endpoint | | Ad hoc cancellation and failure behavior | Explicit lifecycle and terminal error semantics | @@ -108,9 +109,9 @@ The canonical schema is organized by domain under | Non-generative tasks | Typed embedding, classification, and grouped query/candidate scoring with stable correlation | | Structured output | JSON Schema, JSON object, regex, EBNF grammar, structural tags, and fixed choices | | Token information | Prompt and output logprobs, ranks, candidate-token selection, per-token records, and streamed text deltas | -| Discovery | Engine identity, schema revision, role, model limits, topology, parser configuration, and generation capabilities | +| Discovery | Server identity, schema revision, role, model limits, topology, parser configuration, and generation capabilities | | Lifecycle | Health checks, targeted or global abort, graceful drain, progress, and terminal failures | -| Disaggregated serving | Prefill/decode roles, KV session handoff, connector discovery, rank affinity, and cache controls | +| Disaggregated serving | Prefill/decode roles, decode-context parallel topology, KV handoff, connector discovery, and cache controls | | KV-aware routing | Typed KV event streams plus discovery of engine-native event sources | | Model extensions | Multimodal inputs and LoRA adapter lifecycle | | Observability | Point-in-time load snapshots and structured runtime event streams | diff --git a/docs/api.md b/docs/api.md index 427441e..98d2a6b 100644 --- a/docs/api.md +++ b/docs/api.md @@ -20,7 +20,7 @@ package openengine.v1; import "google/protobuf/struct.proto"; -service OpenEngine { +service Inference { // Core inference path. rpc Generate(GenerateRequest) returns (stream GenerateResponse); @@ -28,9 +28,11 @@ service OpenEngine { rpc Embed(EmbedRequest) returns (EmbedResponse); rpc Classify(ClassifyRequest) returns (ClassifyResponse); rpc Score(ScoreRequest) returns (ScoreResponse); +} +service Control { // Runtime metadata and scheduling state. - rpc GetEngineInfo(GetEngineInfoRequest) returns (EngineInfo); + rpc GetServerInfo(GetServerInfoRequest) returns (ServerInfo); rpc GetModelInfo(GetModelInfoRequest) returns (ModelInfo); rpc GetLoad(GetLoadRequest) returns (LoadInfo); @@ -54,9 +56,14 @@ service OpenEngine { } ``` +`Inference` is the inference data plane. `Control` is the discovery, lifecycle, +and coordination control plane. Implementations may expose both services on the +same listener, or isolate them on separate listeners and access policies without +changing the protocol contract. + --- -## Core identity and roles +## Server identity and engine roles ```protobuf enum EngineRole { @@ -66,12 +73,12 @@ enum EngineRole { ENGINE_ROLE_DECODE = 3; } -message GetEngineInfoRequest {} +message GetServerInfoRequest {} -message EngineInfo { +message ServerInfo { string engine_name = 1; // sglang, vllm, tensorrt_llm, etc. string engine_version = 2; - EngineRole role = 3; + EngineRole engine_role = 3; string instance_id = 4; repeated string supported_models = 5; ParallelismInfo parallelism = 6; @@ -87,6 +94,8 @@ message ParallelismInfo { optional uint32 data_parallel_size = 3; optional uint32 data_parallel_rank = 4; optional uint32 data_parallel_start_rank = 5; + // Number of ranks in each decode-context-parallel group. Must be at least 1. + optional uint32 decode_context_parallel_size = 6; } ``` @@ -111,6 +120,12 @@ Discovery response scalars use proto3 `optional` presence. An absent value means the engine cannot report the value; an explicitly present zero or `false` is a reported value and must not be replaced with a client default. +`decode_context_parallel_size` reports the number of ranks across which decode +context is sharded. It describes a group within the server's execution topology +and does not, by itself, imply additional workers beyond the reported tensor, +pipeline, and data-parallel topology. When present, the value must be at least +one; one means decode-context parallelism is disabled. + Role semantics: - `AGGREGATED`: accepts normal generation requests and returns tokens. @@ -192,7 +207,7 @@ enum GuidedDecodingMode { ``` `GetModelInfoRequest.model` is required and selects one of -`EngineInfo.supported_models`; an unknown model returns gRPC `NOT_FOUND`. +`ServerInfo.supported_models`; an unknown model returns gRPC `NOT_FOUND`. Capability submessages distinguish unreported support (message absent) from reported support or lack of support (`supported = true` or `false`). Candidate selection modes and `max_top_n` are reported independently for prompt and @@ -200,7 +215,7 @@ output logprobs. The remaining generation fields advertise support and limits for the corresponding request options. `supports_lora=true` means the engine accepts `GenerateRequest.lora_name` and -the LoRA lifecycle RPCs on `OpenEngine`. +the LoRA lifecycle RPCs on `Control`. --- diff --git a/docs/motivation.md b/docs/motivation.md index c2b695d..8032c48 100644 --- a/docs/motivation.md +++ b/docs/motivation.md @@ -60,9 +60,9 @@ request and control data needed across the process boundary. | Area | Contract | | --- | --- | | Generation | Streaming tokens, usage, finish state, and errors | -| Discovery | Engine, model, role, topology, limits, and capabilities | +| Discovery | Server, model, engine role, topology, limits, and capabilities | | Lifecycle | Health, abort, and drain | -| Scheduling | Load data and data-parallel rank affinity | +| Scheduling | Load data, data-parallel rank affinity, and decode-context parallel topology | | Disaggregation | Prefill readiness, KV-session handoff, and connector data | | KV routing | Event streams and native event-source discovery | diff --git a/proto/openengine/v1/README.md b/proto/openengine/v1/README.md index 6c97034..49b85a5 100644 --- a/proto/openengine/v1/README.md +++ b/proto/openengine/v1/README.md @@ -10,9 +10,9 @@ share the same package and together define the API. | File | Area | | --- | --- | -| [`openengine.proto`](openengine.proto) | `OpenEngine` service and RPC declarations | +| [`openengine.proto`](openengine.proto) | `Inference` and `Control` service declarations | | [`input.proto`](input.proto) | Shared text, token, and multimodal inputs | -| [`engine.proto`](engine.proto) | Engine identity, roles, and parallelism | +| [`server.proto`](server.proto) | Server identity, engine roles, and parallelism | | [`model.proto`](model.proto) | Model metadata and inference capabilities | | [`generation.proto`](generation.proto) | Generation requests, parameters, streamed events, and usage | | [`tasks.proto`](tasks.proto) | Shared non-generative task request and output vocabulary | diff --git a/proto/openengine/v1/lifecycle.proto b/proto/openengine/v1/lifecycle.proto index d183875..25bf42f 100644 --- a/proto/openengine/v1/lifecycle.proto +++ b/proto/openengine/v1/lifecycle.proto @@ -7,9 +7,9 @@ syntax = "proto3"; package openengine.v1; -import "openengine/v1/engine.proto"; import "openengine/v1/error.proto"; import "openengine/v1/kv.proto"; +import "openengine/v1/server.proto"; message HealthRequest { // False means a lightweight readiness/liveness check. True asks the engine to diff --git a/proto/openengine/v1/openengine.proto b/proto/openengine/v1/openengine.proto index 062d68f..8526814 100644 --- a/proto/openengine/v1/openengine.proto +++ b/proto/openengine/v1/openengine.proto @@ -18,7 +18,6 @@ package openengine.v1; import "openengine/v1/classification.proto"; import "openengine/v1/embedding.proto"; -import "openengine/v1/engine.proto"; import "openengine/v1/generation.proto"; import "openengine/v1/kv.proto"; import "openengine/v1/lifecycle.proto"; @@ -26,8 +25,9 @@ import "openengine/v1/lora.proto"; import "openengine/v1/model.proto"; import "openengine/v1/observability.proto"; import "openengine/v1/scoring.proto"; +import "openengine/v1/server.proto"; -service OpenEngine { +service Inference { // Core inference path. rpc Generate(GenerateRequest) returns (stream GenerateResponse); @@ -35,9 +35,11 @@ service OpenEngine { rpc Embed(EmbedRequest) returns (EmbedResponse); rpc Classify(ClassifyRequest) returns (ClassifyResponse); rpc Score(ScoreRequest) returns (ScoreResponse); +} +service Control { // Runtime metadata and scheduling state. - rpc GetEngineInfo(GetEngineInfoRequest) returns (EngineInfo); + rpc GetServerInfo(GetServerInfoRequest) returns (ServerInfo); rpc GetModelInfo(GetModelInfoRequest) returns (ModelInfo); rpc GetLoad(GetLoadRequest) returns (LoadInfo); diff --git a/proto/openengine/v1/engine.proto b/proto/openengine/v1/server.proto similarity index 79% rename from proto/openengine/v1/engine.proto rename to proto/openengine/v1/server.proto index 6766624..dd247a9 100644 --- a/proto/openengine/v1/engine.proto +++ b/proto/openengine/v1/server.proto @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -// Engine identity, role, and parallelism metadata. +// Server identity, engine role, and parallelism metadata. syntax = "proto3"; @@ -16,12 +16,12 @@ enum EngineRole { ENGINE_ROLE_DECODE = 3; } -message GetEngineInfoRequest {} +message GetServerInfoRequest {} -message EngineInfo { +message ServerInfo { string engine_name = 1; // sglang, vllm, tensorrt_llm, etc. string engine_version = 2; - EngineRole role = 3; + EngineRole engine_role = 3; string instance_id = 4; repeated string supported_models = 5; ParallelismInfo parallelism = 6; @@ -37,4 +37,6 @@ message ParallelismInfo { optional uint32 data_parallel_size = 3; optional uint32 data_parallel_rank = 4; optional uint32 data_parallel_start_rank = 5; + // Number of ranks in each decode-context-parallel group. Must be at least 1. + optional uint32 decode_context_parallel_size = 6; }