From f9ed5bf0d9bb0a38e3d6f19efac16e570ee1723e Mon Sep 17 00:00:00 2001 From: Shmuel Kallner Date: Thu, 27 Nov 2025 18:56:40 +0200 Subject: [PATCH 01/15] Added datalayer config to text config definition Signed-off-by: Shmuel Kallner --- .../v1alpha1/endpointpickerconfig_types.go | 39 ++++++++++++++- apix/config/v1alpha1/zz_generated.deepcopy.go | 47 +++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/apix/config/v1alpha1/endpointpickerconfig_types.go b/apix/config/v1alpha1/endpointpickerconfig_types.go index d30f2084c..73961bf7b 100644 --- a/apix/config/v1alpha1/endpointpickerconfig_types.go +++ b/apix/config/v1alpha1/endpointpickerconfig_types.go @@ -50,13 +50,18 @@ type EndpointPickerConfig struct { // SaturationDetector when present specifies the configuration of the // Saturation detector. If not present, default values are used. SaturationDetector *SaturationDetector `json:"saturationDetector,omitempty"` + + // +optional + // Data configures the DataLayer. It is required if the new DataLayer is enabled. + Data *DataLayerConfig `json:"data"` } func (cfg EndpointPickerConfig) String() string { return fmt.Sprintf( - "{Plugins: %v, SchedulingProfiles: %v, FeatureGates: %v, SaturationDetector: %v}", + "{Plugins: %v, SchedulingProfiles: %v, Data: %v, FeatureGates: %v, SaturationDetector: %v}", cfg.Plugins, cfg.SchedulingProfiles, + cfg.Data, cfg.FeatureGates, cfg.SaturationDetector, ) @@ -193,3 +198,35 @@ func (sd *SaturationDetector) String() string { } return "{" + result + "}" } + +// DataLayerConfig contains the configuration of the V2 DataLayer feature +type DataLayerConfig struct { + // +required + // +kubebuilder:validation:Required + // Sources is the list of sources to define to the DataLayer + Sources []DataLayerSource `json:"sources"` +} + +func (dlc DataLayerConfig) String() string { + return fmt.Sprintf("{Sources: %v}", dlc.Sources) +} + +type DataLayerSource struct { + // +required + // +kubebuilder:validation:Required + // PluginRef specifies a partiular Plugin instance to be associated with + // this Source. The reference is to the name of an entry of the Plugins + // defined in the configuration's Plugins section + PluginRef string `json:"pluginRef"` + + // +required + // +kubebuilder:validation:Required + // Extractors specifies the list of Plugin instances to be associated with + // this Source. The entries are references to the names of entries of the Plugins + // defined in the configuration's Plugins section + Extractors []string `json:"extractors"` +} + +func (dls DataLayerSource) String() string { + return fmt.Sprintf("{PluginRef: %s, Extractors: %v}", dls.PluginRef, dls.Extractors) +} diff --git a/apix/config/v1alpha1/zz_generated.deepcopy.go b/apix/config/v1alpha1/zz_generated.deepcopy.go index 701f73793..b61b35645 100644 --- a/apix/config/v1alpha1/zz_generated.deepcopy.go +++ b/apix/config/v1alpha1/zz_generated.deepcopy.go @@ -25,6 +25,48 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataLayerConfig) DeepCopyInto(out *DataLayerConfig) { + *out = *in + if in.Sources != nil { + in, out := &in.Sources, &out.Sources + *out = make([]DataLayerSource, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataLayerConfig. +func (in *DataLayerConfig) DeepCopy() *DataLayerConfig { + if in == nil { + return nil + } + out := new(DataLayerConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataLayerSource) DeepCopyInto(out *DataLayerSource) { + *out = *in + if in.Extractors != nil { + in, out := &in.Extractors, &out.Extractors + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataLayerSource. +func (in *DataLayerSource) DeepCopy() *DataLayerSource { + if in == nil { + return nil + } + out := new(DataLayerSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EndpointPickerConfig) DeepCopyInto(out *EndpointPickerConfig) { *out = *in @@ -53,6 +95,11 @@ func (in *EndpointPickerConfig) DeepCopyInto(out *EndpointPickerConfig) { *out = new(SaturationDetector) **out = **in } + if in.Data != nil { + in, out := &in.Data, &out.Data + *out = new(DataLayerConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EndpointPickerConfig. From 6e547c6bf473ed033577f44b240e533629fca05f Mon Sep 17 00:00:00 2001 From: Shmuel Kallner Date: Thu, 27 Nov 2025 18:57:07 +0200 Subject: [PATCH 02/15] Added datalayer config to EPP config Signed-off-by: Shmuel Kallner --- pkg/epp/config/config.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/epp/config/config.go b/pkg/epp/config/config.go index d966e3c0b..e36812879 100644 --- a/pkg/epp/config/config.go +++ b/pkg/epp/config/config.go @@ -17,6 +17,7 @@ limitations under the License. package config import ( + "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/datalayer" "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/saturationdetector" "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling" ) @@ -25,4 +26,5 @@ import ( type Config struct { SchedulerConfig *scheduling.SchedulerConfig SaturationDetectorConfig *saturationdetector.Config + DataConfig *datalayer.Config } From 24f56a565e2f0002e18344eb7eaee9a53828582b Mon Sep 17 00:00:00 2001 From: Shmuel Kallner Date: Thu, 27 Nov 2025 18:57:43 +0200 Subject: [PATCH 03/15] Load datalayer config from text to EPP config Signed-off-by: Shmuel Kallner --- pkg/epp/config/loader/configloader.go | 42 +++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/pkg/epp/config/loader/configloader.go b/pkg/epp/config/loader/configloader.go index 03126f3b9..b080d32fa 100644 --- a/pkg/epp/config/loader/configloader.go +++ b/pkg/epp/config/loader/configloader.go @@ -28,6 +28,7 @@ import ( configapi "sigs.k8s.io/gateway-api-inference-extension/apix/config/v1alpha1" "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/config" + "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/datalayer" "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/plugins" "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/saturationdetector" "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling" @@ -95,9 +96,15 @@ func InstantiateAndConfigure( return nil, fmt.Errorf("scheduler config build failed: %w", err) } + dataConfig, err := buildDataLayerConfig(rawConfig.Data, rawConfig.FeatureGates, handle) + if err != nil { + return nil, fmt.Errorf("data layer config build failed: %w", err) + } + return &config.Config{ SchedulerConfig: schedulerConfig, SaturationDetectorConfig: buildSaturationConfig(rawConfig.SaturationDetector), + DataConfig: dataConfig, }, nil } @@ -224,3 +231,38 @@ func buildSaturationConfig(apiConfig *configapi.SaturationDetector) *saturationd return cfg } + +func buildDataLayerConfig(rawDataConfig *configapi.DataLayerConfig, rawFeatureGates configapi.FeatureGates, handle plugins.Handle) (*datalayer.Config, error) { + featureGates := loadFeatureConfig(rawFeatureGates) + if !featureGates[datalayer.FeatureGate] { + return nil, nil + } + + if rawDataConfig == nil { + return nil, errors.New("the Datalayer has been enabled. You must specify the Data section in the configuration") + } + + cfg := datalayer.Config{ + Sources: []datalayer.DataSourceConfig{}, + } + for _, source := range rawDataConfig.Sources { + if sourcePlugin, ok := handle.Plugin(source.PluginRef).(datalayer.DataSource); ok { + sourceConfig := datalayer.DataSourceConfig{ + Plugin: sourcePlugin, + Extractors: []datalayer.Extractor{}, + } + for _, extractor := range source.Extractors { + if extractorPlugin, ok := handle.Plugin(extractor).(datalayer.Extractor); ok { + sourceConfig.Extractors = append(sourceConfig.Extractors, extractorPlugin) + } else { + return nil, fmt.Errorf("the plugin %s is not a datalayer.Extractor", source.PluginRef) + } + } + cfg.Sources = append(cfg.Sources, sourceConfig) + } else { + return nil, fmt.Errorf("the plugin %s is not a datalayer.Source", source.PluginRef) + } + } + + return &cfg, nil +} From eae950780fe5fd2ed018fa7ef697004b7f8a6749 Mon Sep 17 00:00:00 2001 From: Shmuel Kallner Date: Thu, 27 Nov 2025 18:58:07 +0200 Subject: [PATCH 04/15] Updated tests for datalayer config Signed-off-by: Shmuel Kallner --- pkg/epp/config/loader/configloader_test.go | 52 ++++++++++++++++ pkg/epp/config/loader/testdata_test.go | 69 ++++++++++++++++++++++ test/testdata/configloader_1_test.yaml | 7 +++ 3 files changed, 128 insertions(+) diff --git a/pkg/epp/config/loader/configloader_test.go b/pkg/epp/config/loader/configloader_test.go index 82f57f4b2..32e0f42e5 100644 --- a/pkg/epp/config/loader/configloader_test.go +++ b/pkg/epp/config/loader/configloader_test.go @@ -19,6 +19,7 @@ package loader import ( "context" "encoding/json" + "reflect" "testing" "time" @@ -47,6 +48,8 @@ const ( testPickerType = "test-picker" testScorerType = "test-scorer" testProfileHandler = "test-profile-handler" + testSourceType = "test-source" + testExtractorType = "test-extractor" ) // --- Test: Phase 1 (Raw Loading & Static Defaults) --- @@ -281,6 +284,21 @@ func TestInstantiateAndConfigure(t *testing.T) { configText: errorMultiProfilesUseSingleProfileHandlerText, wantErr: true, }, + { + name: "Error - Missing Data Config", + configText: errorMissingDataConfigText, + wantErr: true, + }, + { + name: "Error - Bad Source Reference", + configText: errorBadSourceReferenceText, + wantErr: true, + }, + { + name: "Error - Bad Extractor Reference", + configText: errorBadExtractorReferenceText, + wantErr: true, + }, } for _, tc := range tests { @@ -422,6 +440,32 @@ func (m *mockHandler) ProcessResults( return nil, nil } +// Mock Source +type mockSource struct{ mockPlugin } + +func (m *mockSource) AddExtractor(_ datalayer.Extractor) error { + return nil +} + +func (m *mockSource) Collect(ctx context.Context, ep datalayer.Endpoint) error { + return nil +} + +func (m *mockSource) Extractors() []string { + return []string{} +} + +// Mock Extractor +type mockExtractor struct{ mockPlugin } + +func (m *mockExtractor) ExpectedInputType() reflect.Type { + return reflect.TypeOf("") +} + +func (m *mockExtractor) Extract(ctx context.Context, data any, ep datalayer.Endpoint) error { + return nil +} + func registerTestPlugins(t *testing.T) { t.Helper() @@ -460,6 +504,14 @@ func registerTestPlugins(t *testing.T) { return &mockHandler{mockPlugin{t: plugins.TypedName{Name: name, Type: testProfileHandler}}}, nil }) + plugins.Register(testSourceType, func(name string, _ json.RawMessage, _ plugins.Handle) (plugins.Plugin, error) { + return &mockSource{mockPlugin{t: plugins.TypedName{Name: name, Type: testSourceType}}}, nil + }) + + plugins.Register(testExtractorType, func(name string, _ json.RawMessage, _ plugins.Handle) (plugins.Plugin, error) { + return &mockExtractor{mockPlugin{t: plugins.TypedName{Name: name, Type: testExtractorType}}}, nil + }) + // Ensure system defaults are registered too. plugins.Register(picker.MaxScorePickerType, picker.MaxScorePickerFactory) plugins.Register(profile.SingleProfileHandlerType, profile.SingleProfileHandlerFactory) diff --git a/pkg/epp/config/loader/testdata_test.go b/pkg/epp/config/loader/testdata_test.go index a2e8adf23..9902f6b1d 100644 --- a/pkg/epp/config/loader/testdata_test.go +++ b/pkg/epp/config/loader/testdata_test.go @@ -75,12 +75,21 @@ plugins: type: max-score-picker - name: profileHandler type: single-profile-handler +- name: testSource + type: test-source +- name: testExtractor + type: test-extractor schedulingProfiles: - name: default plugins: - pluginRef: testScorer weight: 50 - pluginRef: maxScorePicker +data: + sources: + - pluginRef: testSource + extractors: + - testExtractor featureGates: - dataLayer ` @@ -349,3 +358,63 @@ schedulingProfiles: plugins: - pluginRef: maxScore ` + +// errorMissingDataConfigText has the datalayer enabled without config +const errorMissingDataConfigText = ` +apiVersion: inference.networking.x-k8s.io/v1alpha1 +kind: EndpointPickerConfig +plugins: +- name: test1 + type: test-one + parameters: + threshold: 10 +schedulingProfiles: +- name: default + plugins: + - pluginRef: test1 +featureGates: +- dataLayer +` + +// errorBadSourceReferenceText has a bad DataSource plugin reference +const errorBadSourceReferenceText = ` +apiVersion: inference.networking.x-k8s.io/v1alpha1 +kind: EndpointPickerConfig +plugins: +- name: test1 + type: test-one + parameters: + threshold: 10 +schedulingProfiles: +- name: default + plugins: + - pluginRef: test1 +data: + sources: + - pluginRef: test-one +featureGates: +- dataLayer +` + +// errorBadExtractorReferenceText has a bad Extractor plugin reference +const errorBadExtractorReferenceText = ` +apiVersion: inference.networking.x-k8s.io/v1alpha1 +kind: EndpointPickerConfig +plugins: +- name: test1 + type: test-one + parameters: + threshold: 10 +- type: test-source +schedulingProfiles: +- name: default + plugins: + - pluginRef: test1 +data: + sources: + - pluginRef: test-source + extractors: + - test-one +featureGates: +- dataLayer +` diff --git a/test/testdata/configloader_1_test.yaml b/test/testdata/configloader_1_test.yaml index f43684158..fd2bc5555 100644 --- a/test/testdata/configloader_1_test.yaml +++ b/test/testdata/configloader_1_test.yaml @@ -12,6 +12,8 @@ plugins: blockSize: 32 - name: testPicker type: test-picker +- type: test-source +- type: test-extractor schedulingProfiles: - name: default plugins: @@ -19,6 +21,11 @@ schedulingProfiles: - pluginRef: test-two weight: 50 - pluginRef: testPicker +data: + sources: + - pluginRef: test-source + extractors: + - test-extractor featureGates: - dataLayer saturationDetector: From 89b07bfeb100dacbae03a258246b5811274c9353 Mon Sep 17 00:00:00 2001 From: Shmuel Kallner Date: Mon, 1 Dec 2025 18:29:56 +0200 Subject: [PATCH 05/15] Updated configuration documentation Signed-off-by: Shmuel Kallner --- .../guides/epp-configuration/config-text.md | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/site-src/guides/epp-configuration/config-text.md b/site-src/guides/epp-configuration/config-text.md index 252609102..f486b769c 100644 --- a/site-src/guides/epp-configuration/config-text.md +++ b/site-src/guides/epp-configuration/config-text.md @@ -26,6 +26,8 @@ schedulingProfiles: - .... saturationDetector: ... +data: + ... featureGates: ... ``` @@ -40,6 +42,9 @@ requests to pods. This section is described in more detail in the section [Confi The saturationDetector section configures the saturation detector, which is used to determine if special action needs to eb taken due to the system being overloaded or saturated. This section is described in more detail in the section [Saturation Detector configuration](#saturation-detector-configuration) +The data section configures the data layer, which is used to gather metrics and other data used in making scheduling decisions. +This section is described in more detail in the section [Data Layer configuration](#data-layer-configuration) + The featureGates sections allows the enablement of experimental features of the IGW. This section is described in more detail in the section [Feature Gates](#feature-gates) @@ -330,6 +335,34 @@ a value of `0.8` will be used. metrics are older than this, it might be excluded from "good capacity" considerations or treated as having no capacity for safety. This field is optional, if omitted a value of `200ms` will be used. +## Data Layer configuration + +The Data Layer collects metrics and other data used in scheduling decisions made by the various configured +filters and plugins. The exact data collected varies by the DataSource and Extractors configured. The basic ones +collect Prometheus metrics from the Model Servers in the InferencePool. + +The Data Layer is configured via the data section of the overall configuration. It has the following form: + +```yaml +data: + sources: + - pluginRef: source1 + extractors: + - extarctor1 + - extractor2 +``` + +The data section has one field *sources* which configures the set of DataSources to be used to gather the metrics +and other data used for scheduling. + +Each entry in the sources list has the following fields: + +- *pluginRef* is a reference to the name of the plugin instance to be used. +- *extractors* specifies the list of the extractor plugin instances, by name, to be used with this DataSource. + +**Note**: The names of the plugin instances mentioned above, refer to plugin instances defined in the plugins section +of the configuration. + ## Feature Gates The Feature Gates section allows for the enabling of experimental features of the IGW. These experimental From 0afd2357ac98be8ffb03d35806ac18378e5bf5a3 Mon Sep 17 00:00:00 2001 From: Shmuel Kallner Date: Tue, 2 Dec 2025 15:45:27 +0200 Subject: [PATCH 06/15] Refactored configuration DataLayer Extractor elements Signed-off-by: Shmuel Kallner --- .../v1alpha1/endpointpickerconfig_types.go | 31 ++++++++++++++----- apix/config/v1alpha1/zz_generated.deepcopy.go | 27 ++++++++++++---- pkg/epp/config/loader/configloader.go | 2 +- pkg/epp/config/loader/testdata_test.go | 2 +- 4 files changed, 46 insertions(+), 16 deletions(-) diff --git a/apix/config/v1alpha1/endpointpickerconfig_types.go b/apix/config/v1alpha1/endpointpickerconfig_types.go index 73961bf7b..29040fdb3 100644 --- a/apix/config/v1alpha1/endpointpickerconfig_types.go +++ b/apix/config/v1alpha1/endpointpickerconfig_types.go @@ -30,6 +30,11 @@ import ( type EndpointPickerConfig struct { metav1.TypeMeta `json:",inline"` + // +optional + // FeatureGates is a set of flags that enable various experimental features with the EPP. + // If omitted non of these experimental features will be enabled. + FeatureGates FeatureGates `json:"featureGates,omitempty"` + // +required // +kubebuilder:validation:Required // Plugins is the list of plugins that will be instantiated. @@ -41,11 +46,6 @@ type EndpointPickerConfig struct { // that will be created. SchedulingProfiles []SchedulingProfile `json:"schedulingProfiles"` - // +optional - // FeatureGates is a set of flags that enable various experimental features with the EPP. - // If omitted non of these experimental features will be enabled. - FeatureGates FeatureGates `json:"featureGates,omitempty"` - // +optional // SaturationDetector when present specifies the configuration of the // Saturation detector. If not present, default values are used. @@ -58,11 +58,11 @@ type EndpointPickerConfig struct { func (cfg EndpointPickerConfig) String() string { return fmt.Sprintf( - "{Plugins: %v, SchedulingProfiles: %v, Data: %v, FeatureGates: %v, SaturationDetector: %v}", + "{FeatureGates: %v, Plugins: %v, SchedulingProfiles: %v, Data: %v, SaturationDetector: %v}", + cfg.FeatureGates, cfg.Plugins, cfg.SchedulingProfiles, cfg.Data, - cfg.FeatureGates, cfg.SaturationDetector, ) } @@ -211,6 +211,7 @@ func (dlc DataLayerConfig) String() string { return fmt.Sprintf("{Sources: %v}", dlc.Sources) } +// DataLayerSource contains the configuration of a DataSource of the V2 DataLayer feature type DataLayerSource struct { // +required // +kubebuilder:validation:Required @@ -224,9 +225,23 @@ type DataLayerSource struct { // Extractors specifies the list of Plugin instances to be associated with // this Source. The entries are references to the names of entries of the Plugins // defined in the configuration's Plugins section - Extractors []string `json:"extractors"` + Extractors []DataLayerExtractor `json:"extractors"` } func (dls DataLayerSource) String() string { return fmt.Sprintf("{PluginRef: %s, Extractors: %v}", dls.PluginRef, dls.Extractors) } + +// DataLayerExtractor contains the configuration of an Extractor of the V2 DataLayer feature +type DataLayerExtractor struct { + // +required + // +kubebuilder:validation:Required + // PluginRef specifies a partiular Plugin instance to be associated with + // this Extractor. The reference is to the name of an entry of the Plugins + // defined in the configuration's Plugins section + PluginRef string `json:"pluginRef"` +} + +func (dle DataLayerExtractor) String() string { + return "{PluginRef: " + dle.PluginRef + "}" +} diff --git a/apix/config/v1alpha1/zz_generated.deepcopy.go b/apix/config/v1alpha1/zz_generated.deepcopy.go index b61b35645..dec218e87 100644 --- a/apix/config/v1alpha1/zz_generated.deepcopy.go +++ b/apix/config/v1alpha1/zz_generated.deepcopy.go @@ -47,12 +47,27 @@ func (in *DataLayerConfig) DeepCopy() *DataLayerConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataLayerExtractor) DeepCopyInto(out *DataLayerExtractor) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataLayerExtractor. +func (in *DataLayerExtractor) DeepCopy() *DataLayerExtractor { + if in == nil { + return nil + } + out := new(DataLayerExtractor) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DataLayerSource) DeepCopyInto(out *DataLayerSource) { *out = *in if in.Extractors != nil { in, out := &in.Extractors, &out.Extractors - *out = make([]string, len(*in)) + *out = make([]DataLayerExtractor, len(*in)) copy(*out, *in) } } @@ -71,6 +86,11 @@ func (in *DataLayerSource) DeepCopy() *DataLayerSource { func (in *EndpointPickerConfig) DeepCopyInto(out *EndpointPickerConfig) { *out = *in out.TypeMeta = in.TypeMeta + if in.FeatureGates != nil { + in, out := &in.FeatureGates, &out.FeatureGates + *out = make(FeatureGates, len(*in)) + copy(*out, *in) + } if in.Plugins != nil { in, out := &in.Plugins, &out.Plugins *out = make([]PluginSpec, len(*in)) @@ -85,11 +105,6 @@ func (in *EndpointPickerConfig) DeepCopyInto(out *EndpointPickerConfig) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.FeatureGates != nil { - in, out := &in.FeatureGates, &out.FeatureGates - *out = make(FeatureGates, len(*in)) - copy(*out, *in) - } if in.SaturationDetector != nil { in, out := &in.SaturationDetector, &out.SaturationDetector *out = new(SaturationDetector) diff --git a/pkg/epp/config/loader/configloader.go b/pkg/epp/config/loader/configloader.go index b080d32fa..1ae92a807 100644 --- a/pkg/epp/config/loader/configloader.go +++ b/pkg/epp/config/loader/configloader.go @@ -252,7 +252,7 @@ func buildDataLayerConfig(rawDataConfig *configapi.DataLayerConfig, rawFeatureGa Extractors: []datalayer.Extractor{}, } for _, extractor := range source.Extractors { - if extractorPlugin, ok := handle.Plugin(extractor).(datalayer.Extractor); ok { + if extractorPlugin, ok := handle.Plugin(extractor.PluginRef).(datalayer.Extractor); ok { sourceConfig.Extractors = append(sourceConfig.Extractors, extractorPlugin) } else { return nil, fmt.Errorf("the plugin %s is not a datalayer.Extractor", source.PluginRef) diff --git a/pkg/epp/config/loader/testdata_test.go b/pkg/epp/config/loader/testdata_test.go index 9902f6b1d..f3cc98cf3 100644 --- a/pkg/epp/config/loader/testdata_test.go +++ b/pkg/epp/config/loader/testdata_test.go @@ -89,7 +89,7 @@ data: sources: - pluginRef: testSource extractors: - - testExtractor + - pluginRef: testExtractor featureGates: - dataLayer ` From fc1747cbae704bc59ec7e559ca25e46cdba2f84b Mon Sep 17 00:00:00 2001 From: Shmuel Kallner Date: Tue, 2 Dec 2025 15:46:09 +0200 Subject: [PATCH 07/15] Test data updates due to configuration refactoring Signed-off-by: Shmuel Kallner --- test/testdata/configloader_1_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/testdata/configloader_1_test.yaml b/test/testdata/configloader_1_test.yaml index fd2bc5555..2d724d7b7 100644 --- a/test/testdata/configloader_1_test.yaml +++ b/test/testdata/configloader_1_test.yaml @@ -25,7 +25,7 @@ data: sources: - pluginRef: test-source extractors: - - test-extractor + - pluginRef: test-extractor featureGates: - dataLayer saturationDetector: From 20b1106da23d512a6815f6499e2ab96e2d138965 Mon Sep 17 00:00:00 2001 From: Shmuel Kallner Date: Tue, 2 Dec 2025 15:47:01 +0200 Subject: [PATCH 08/15] Documentation updates due to comments from the review Signed-off-by: Shmuel Kallner --- .../guides/epp-configuration/config-text.md | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/site-src/guides/epp-configuration/config-text.md b/site-src/guides/epp-configuration/config-text.md index f486b769c..88c6e32dc 100644 --- a/site-src/guides/epp-configuration/config-text.md +++ b/site-src/guides/epp-configuration/config-text.md @@ -34,19 +34,19 @@ featureGates: The first two lines of the configuration are constant and must appear as is. -The plugins section defines the set of plugins that will be instantiated and their parameters. This section is described in more detail in the section [Configuring Plugins via text](#configuring-plugins-via-text) +The `featureGates` section allows the enablement of experimental features of the IGW. This section is +described in more detail in the section [Feature Gates](#feature-gates) + +The `plugins` section defines the set of plugins that will be instantiated and their parameters. This section is described in more detail in the section [Configuring Plugins via text](#configuring-plugins-via-text) -The schedulingProfiles section defines the set of scheduling profiles that can be used in scheduling +The `schedulingProfiles` section defines the set of scheduling profiles that can be used in scheduling requests to pods. This section is described in more detail in the section [Configuring Plugins via YAML](#configuring-plugins-via-yaml) -The saturationDetector section configures the saturation detector, which is used to determine if special +The `saturationDetector` section configures the saturation detector, which is used to determine if special action needs to eb taken due to the system being overloaded or saturated. This section is described in more detail in the section [Saturation Detector configuration](#saturation-detector-configuration) -The data section configures the data layer, which is used to gather metrics and other data used in making scheduling decisions. -This section is described in more detail in the section [Data Layer configuration](#data-layer-configuration) - -The featureGates sections allows the enablement of experimental features of the IGW. This section is -described in more detail in the section [Feature Gates](#feature-gates) +The `data` section configures the data layer, which is used to gather information (such as metrics) used in making scheduling +decisions. This section is described in more detail in the section [Data Layer configuration](#data-layer-configuration) ## Configuring Plugins via YAML @@ -69,7 +69,7 @@ In addition, the set of instantiated plugins can also include a picker, which ch the request is scheduled after filtering and scoring. If one is not referenced in a SchedulingProfile, an instance of `MaxScorePicker` will be added to the SchedulingProfile in question. -The plugins section defines the set of plugins that will be instantiated and their parameters. +The `plugins` section defines the set of plugins that will be instantiated and their parameters. Each entry in this section has the following form: ```yaml @@ -88,7 +88,7 @@ field is omitted, the plugin's type will be used as its name. - *parameters* which is optional, defines the set of parameters used to configure the plugin in question. The actual set of parameters varies from plugin to plugin. -The schedulingProfiles section defines the set of scheduling profiles that can be used in scheduling +The `schedulingProfiles` section defines the set of scheduling profiles that can be used in scheduling requests to pods. The number of scheduling profiles one defines, depends on the use case. For simple serving of requests, one is enough. For disaggregated prefill, two profiles are required. Each entry in this section has the following form: @@ -313,7 +313,7 @@ The Saturation Detector determines that the cluster is saturated by looking at t - KV cache utilization - Metrics staleness -The Saturation Detector is configured via the saturationDetector section of the overall configuration. +The Saturation Detector is configured via the `saturationDetector` section of the overall configuration. It has the following form: ```yaml @@ -323,7 +323,7 @@ saturationDetector: metricsStalenessThreshold: 150ms ``` -The various sub-fields of the saturationDetector section are: +The various sub-fields of the `saturationDetector` section are: - The `queueDepthThreshold` field which defines the backend waiting queue size above which a pod is considered to have insufficient capacity for new requests. This field is optional, if @@ -338,18 +338,18 @@ as having no capacity for safety. This field is optional, if omitted a value of ## Data Layer configuration The Data Layer collects metrics and other data used in scheduling decisions made by the various configured -filters and plugins. The exact data collected varies by the DataSource and Extractors configured. The basic ones -collect Prometheus metrics from the Model Servers in the InferencePool. +plugins. The exact data collected varies by the DataSource and Extractors configured. The baseline +provided in GAIE collect Prometheus metrics from the Model Servers in the InferencePool. -The Data Layer is configured via the data section of the overall configuration. It has the following form: +The Data Layer is configured via the `data` section of the overall configuration. It has the following form: ```yaml data: sources: - pluginRef: source1 extractors: - - extarctor1 - - extractor2 + - pluginRef: extractor1 + - pluginRef: extractor2 ``` The data section has one field *sources* which configures the set of DataSources to be used to gather the metrics @@ -358,7 +358,9 @@ and other data used for scheduling. Each entry in the sources list has the following fields: - *pluginRef* is a reference to the name of the plugin instance to be used. -- *extractors* specifies the list of the extractor plugin instances, by name, to be used with this DataSource. +- *extractors* specifies the list of the extractors to be used with this DataSource. Each entry in the extractors +list has the following field: + - *pluginRef* is a reference to the name of the plugin instances to be used. **Note**: The names of the plugin instances mentioned above, refer to plugin instances defined in the plugins section of the configuration. From 0d03c6e871821282261906224559d4ef3f3c95ed Mon Sep 17 00:00:00 2001 From: Shmuel Kallner Date: Thu, 4 Dec 2025 14:19:30 +0200 Subject: [PATCH 09/15] Removed V2 from errors and descriptions WRT the Datalayer Signed-off-by: Shmuel Kallner --- apix/config/v1alpha1/endpointpickerconfig_types.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apix/config/v1alpha1/endpointpickerconfig_types.go b/apix/config/v1alpha1/endpointpickerconfig_types.go index 29040fdb3..18898ce88 100644 --- a/apix/config/v1alpha1/endpointpickerconfig_types.go +++ b/apix/config/v1alpha1/endpointpickerconfig_types.go @@ -199,7 +199,7 @@ func (sd *SaturationDetector) String() string { return "{" + result + "}" } -// DataLayerConfig contains the configuration of the V2 DataLayer feature +// DataLayerConfig contains the configuration of the DataLayer feature type DataLayerConfig struct { // +required // +kubebuilder:validation:Required @@ -211,7 +211,7 @@ func (dlc DataLayerConfig) String() string { return fmt.Sprintf("{Sources: %v}", dlc.Sources) } -// DataLayerSource contains the configuration of a DataSource of the V2 DataLayer feature +// DataLayerSource contains the configuration of a DataSource of the DataLayer feature type DataLayerSource struct { // +required // +kubebuilder:validation:Required @@ -232,7 +232,7 @@ func (dls DataLayerSource) String() string { return fmt.Sprintf("{PluginRef: %s, Extractors: %v}", dls.PluginRef, dls.Extractors) } -// DataLayerExtractor contains the configuration of an Extractor of the V2 DataLayer feature +// DataLayerExtractor contains the configuration of an Extractor of the DataLayer feature type DataLayerExtractor struct { // +required // +kubebuilder:validation:Required From 69554558869a17c17c68e0cc214ab3713cf15ab5 Mon Sep 17 00:00:00 2001 From: Shmuel Kallner Date: Thu, 4 Dec 2025 14:20:10 +0200 Subject: [PATCH 10/15] Ensure there is no Datalayer config if it isn't enabled Signed-off-by: Shmuel Kallner --- pkg/epp/config/loader/configloader.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/epp/config/loader/configloader.go b/pkg/epp/config/loader/configloader.go index 1ae92a807..a69e61ace 100644 --- a/pkg/epp/config/loader/configloader.go +++ b/pkg/epp/config/loader/configloader.go @@ -235,6 +235,9 @@ func buildSaturationConfig(apiConfig *configapi.SaturationDetector) *saturationd func buildDataLayerConfig(rawDataConfig *configapi.DataLayerConfig, rawFeatureGates configapi.FeatureGates, handle plugins.Handle) (*datalayer.Config, error) { featureGates := loadFeatureConfig(rawFeatureGates) if !featureGates[datalayer.FeatureGate] { + if rawDataConfig != nil { + return nil, errors.New("the Datalayer has not been enabled, but you specified a configuration for it") + } return nil, nil } From 5db0477c706c64da6bd2e4a80c5c20a4673d5d1e Mon Sep 17 00:00:00 2001 From: Shmuel Kallner Date: Thu, 4 Dec 2025 14:20:50 +0200 Subject: [PATCH 11/15] Debug build failure Signed-off-by: Shmuel Kallner --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index b9963e531..d59abb319 100644 --- a/Makefile +++ b/Makefile @@ -466,6 +466,7 @@ $(GCI): $(LOCALBIN) # $2 - package url which can be installed # $3 - specific version of package define go-install-tool +set -x; \ @[ -f "$(1)-$(3)" ] || { \ set -e; \ package=$(2)@$(3) ;\ From d2af438a1b9ccdb39d84a6cea67dece66a8fb8a9 Mon Sep 17 00:00:00 2001 From: Shmuel Kallner Date: Thu, 4 Dec 2025 15:00:34 +0200 Subject: [PATCH 12/15] Removed debugging statement Signed-off-by: Shmuel Kallner --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index d59abb319..b9963e531 100644 --- a/Makefile +++ b/Makefile @@ -466,7 +466,6 @@ $(GCI): $(LOCALBIN) # $2 - package url which can be installed # $3 - specific version of package define go-install-tool -set -x; \ @[ -f "$(1)-$(3)" ] || { \ set -e; \ package=$(2)@$(3) ;\ From 7ee4c710b6c0959b63476a865305079250b60ee4 Mon Sep 17 00:00:00 2001 From: Shmuel Kallner Date: Thu, 4 Dec 2025 15:10:18 +0200 Subject: [PATCH 13/15] Add verbose to failing command Signed-off-by: Shmuel Kallner --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b9963e531..cb85e494e 100644 --- a/Makefile +++ b/Makefile @@ -440,7 +440,7 @@ $(GOLANGCI_LINT): $(LOCALBIN) .PHONY: golangci-api-lint golangci-api-lint: golangci-lint $(GOLANGCI_API_LINT) ## Download golangci-lint locally if necessary before building KAL $(GOLANGCI_API_LINT): - $(GOLANGCI_LINT) custom + $(GOLANGCI_LINT) -v custom .PHONY: yq yq: ## Download yq locally if necessary. From 972d9666763564db4f0247ea094e9691340eaf05 Mon Sep 17 00:00:00 2001 From: Shmuel Kallner Date: Thu, 4 Dec 2025 15:24:03 +0200 Subject: [PATCH 14/15] Remove verbose from formerly failing command Signed-off-by: Shmuel Kallner --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index cb85e494e..b9963e531 100644 --- a/Makefile +++ b/Makefile @@ -440,7 +440,7 @@ $(GOLANGCI_LINT): $(LOCALBIN) .PHONY: golangci-api-lint golangci-api-lint: golangci-lint $(GOLANGCI_API_LINT) ## Download golangci-lint locally if necessary before building KAL $(GOLANGCI_API_LINT): - $(GOLANGCI_LINT) -v custom + $(GOLANGCI_LINT) custom .PHONY: yq yq: ## Download yq locally if necessary. From 94ff56c9714f3c184b4f0bbe719ca7a274b264b2 Mon Sep 17 00:00:00 2001 From: Shmuel Kallner Date: Wed, 10 Dec 2025 15:10:30 +0200 Subject: [PATCH 15/15] Update from review comment Signed-off-by: Shmuel Kallner --- pkg/epp/config/loader/configloader.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/epp/config/loader/configloader.go b/pkg/epp/config/loader/configloader.go index a69e61ace..71e57bf08 100644 --- a/pkg/epp/config/loader/configloader.go +++ b/pkg/epp/config/loader/configloader.go @@ -96,7 +96,8 @@ func InstantiateAndConfigure( return nil, fmt.Errorf("scheduler config build failed: %w", err) } - dataConfig, err := buildDataLayerConfig(rawConfig.Data, rawConfig.FeatureGates, handle) + featureGates := loadFeatureConfig(rawConfig.FeatureGates) + dataConfig, err := buildDataLayerConfig(rawConfig.Data, featureGates[datalayer.FeatureGate], handle) if err != nil { return nil, fmt.Errorf("data layer config build failed: %w", err) } @@ -232,9 +233,8 @@ func buildSaturationConfig(apiConfig *configapi.SaturationDetector) *saturationd return cfg } -func buildDataLayerConfig(rawDataConfig *configapi.DataLayerConfig, rawFeatureGates configapi.FeatureGates, handle plugins.Handle) (*datalayer.Config, error) { - featureGates := loadFeatureConfig(rawFeatureGates) - if !featureGates[datalayer.FeatureGate] { +func buildDataLayerConfig(rawDataConfig *configapi.DataLayerConfig, dataLayerEnabled bool, handle plugins.Handle) (*datalayer.Config, error) { + if !dataLayerEnabled { if rawDataConfig != nil { return nil, errors.New("the Datalayer has not been enabled, but you specified a configuration for it") }