Skip to content

Commit e97d1a0

Browse files
ngopalak-redhatngopalak-redhat
committed
OCPNODE-3973: Disable autoSizingReserved on control plane and arbiter nodes
OCPNODE-3719: Default Enablement of Auto Sizing Reserved in OpenShift 4.21 - Enable AutoSizingReserved by default for worker nodes - Disable AutoSizingReserved for master/control-plane nodes - Disable AutoSizingReserved for arbiter nodes - Disable AutoSizingReserved for Hypershift clusters - Add corresponding tests for the new behavior This combines changes from PR #5390.
1 parent ae23a39 commit e97d1a0

File tree

8 files changed

+238
-2
lines changed

8 files changed

+238
-2
lines changed

‎pkg/controller/common/constants.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,9 @@ const (
149149
// Stub Ignition upgrade related annotation keys
150150
StubIgnitionVersionAnnotation = "machineconfiguration.openshift.io/stub-ignition-upgraded-to"
151151
StubIgnitionTimestampAnnotation = "machineconfiguration.openshift.io/stub-ignition-upgraded-at"
152+
153+
// NodeSizingEnabledEnvPath is the file path for the node sizing enabled environment file
154+
NodeSizingEnabledEnvPath = "/etc/node-sizing-enabled.env"
152155
)
153156

154157
// Commonly-used MCO ConfigMap names

‎pkg/controller/kubelet-config/helpers.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ func createNewKubeletDynamicSystemReservedIgnition(autoSystemReserved *bool, use
6868
config := fmt.Sprintf("NODE_SIZING_ENABLED=%s\nSYSTEM_RESERVED_MEMORY=%s\nSYSTEM_RESERVED_CPU=%s\nSYSTEM_RESERVED_ES=%s\n",
6969
autoNodeSizing, systemReservedMemory, systemReservedCPU, systemReservedEphemeralStorage)
7070

71-
r := ctrlcommon.NewIgnFileBytesOverwriting("/etc/node-sizing-enabled.env", []byte(config))
71+
r := ctrlcommon.NewIgnFileBytesOverwriting(ctrlcommon.NodeSizingEnabledEnvPath, []byte(config))
7272
return &r
7373
}
7474

‎pkg/controller/template/template_controller_test.go‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package template
33
import (
44
"fmt"
55
"reflect"
6+
"strings"
67
"testing"
78
"time"
89

@@ -448,3 +449,127 @@ func getKey(config *mcfgv1.ControllerConfig, t *testing.T) string {
448449
}
449450
return key
450451
}
452+
453+
func TestKubeletAutoNodeSizingEnabled(t *testing.T) {
454+
cc := newControllerConfig("test-cluster")
455+
ps := []byte(`{"dummy": "dummy"}`)
456+
457+
mcs, err := getMachineConfigsForControllerConfig(templateDir, cc, ps, nil)
458+
if err != nil {
459+
t.Fatal(err)
460+
}
461+
462+
// Find machine configs that should contain the auto-node-sizing file
463+
// The file should be in all role-based machine configs (master, worker)
464+
autoSizingFileFound := false
465+
for _, mc := range mcs {
466+
ignCfg, err := ctrlcommon.ParseAndConvertConfig(mc.Spec.Config.Raw)
467+
if err != nil {
468+
t.Fatalf("Failed to parse ignition config for %s: %v", mc.Name, err)
469+
}
470+
471+
// Look for the auto-node-sizing file
472+
for _, file := range ignCfg.Storage.Files {
473+
if file.Path == ctrlcommon.NodeSizingEnabledEnvPath {
474+
autoSizingFileFound = true
475+
476+
// Decode the file contents
477+
contents, err := ctrlcommon.DecodeIgnitionFileContents(file.Contents.Source, file.Contents.Compression)
478+
if err != nil {
479+
t.Fatalf("Failed to decode auto-node-sizing file contents: %v", err)
480+
}
481+
482+
contentsStr := string(contents)
483+
484+
// Verify NODE_SIZING_ENABLED based on node role
485+
// Master nodes should have NODE_SIZING_ENABLED=false
486+
// Other nodes (worker, etc.) should have NODE_SIZING_ENABLED=true
487+
isMasterNode := strings.Contains(mc.Name, "master")
488+
if isMasterNode {
489+
if !strings.Contains(contentsStr, "NODE_SIZING_ENABLED=false") {
490+
t.Errorf("Expected NODE_SIZING_ENABLED=false in %s, got: %s", mc.Name, contentsStr)
491+
}
492+
} else {
493+
if !strings.Contains(contentsStr, "NODE_SIZING_ENABLED=true") {
494+
t.Errorf("Expected NODE_SIZING_ENABLED=true in %s, got: %s", mc.Name, contentsStr)
495+
}
496+
}
497+
498+
// Verify other expected values
499+
if !strings.Contains(contentsStr, "SYSTEM_RESERVED_MEMORY=1Gi") {
500+
t.Errorf("Expected SYSTEM_RESERVED_MEMORY=1Gi in %s, got: %s", mc.Name, contentsStr)
501+
}
502+
503+
if !strings.Contains(contentsStr, "SYSTEM_RESERVED_CPU=500m") {
504+
t.Errorf("Expected SYSTEM_RESERVED_CPU=500m in %s, got: %s", mc.Name, contentsStr)
505+
}
506+
507+
if !strings.Contains(contentsStr, "SYSTEM_RESERVED_ES=1Gi") {
508+
t.Errorf("Expected SYSTEM_RESERVED_ES=1Gi in %s, got: %s", mc.Name, contentsStr)
509+
}
510+
}
511+
}
512+
}
513+
514+
if !autoSizingFileFound {
515+
t.Errorf("Expected to find %s file in at least one machine config", ctrlcommon.NodeSizingEnabledEnvPath)
516+
}
517+
}
518+
519+
func TestKubeletAutoNodeSizingDisabledForHypershift(t *testing.T) {
520+
cc := newControllerConfig("test-cluster")
521+
// Set ControlPlaneTopology to External to simulate Hypershift
522+
cc.Spec.Infra.Status.ControlPlaneTopology = configv1.ExternalTopologyMode
523+
ps := []byte(`{"dummy": "dummy"}`)
524+
525+
mcs, err := getMachineConfigsForControllerConfig(templateDir, cc, ps, nil)
526+
if err != nil {
527+
t.Fatal(err)
528+
}
529+
530+
// Find machine configs that should contain the auto-node-sizing file
531+
autoSizingFileFound := false
532+
for _, mc := range mcs {
533+
ignCfg, err := ctrlcommon.ParseAndConvertConfig(mc.Spec.Config.Raw)
534+
if err != nil {
535+
t.Fatalf("Failed to parse ignition config for %s: %v", mc.Name, err)
536+
}
537+
538+
// Look for the auto-node-sizing file
539+
for _, file := range ignCfg.Storage.Files {
540+
if file.Path == ctrlcommon.NodeSizingEnabledEnvPath {
541+
autoSizingFileFound = true
542+
543+
// Decode the file contents
544+
contents, err := ctrlcommon.DecodeIgnitionFileContents(file.Contents.Source, file.Contents.Compression)
545+
if err != nil {
546+
t.Fatalf("Failed to decode auto-node-sizing file contents: %v", err)
547+
}
548+
549+
contentsStr := string(contents)
550+
551+
// Verify NODE_SIZING_ENABLED=false is present for Hypershift
552+
if !strings.Contains(contentsStr, "NODE_SIZING_ENABLED=false") {
553+
t.Errorf("Expected NODE_SIZING_ENABLED=false for Hypershift in %s, got: %s", mc.Name, contentsStr)
554+
}
555+
556+
// Verify other expected values are still present
557+
if !strings.Contains(contentsStr, "SYSTEM_RESERVED_MEMORY=1Gi") {
558+
t.Errorf("Expected SYSTEM_RESERVED_MEMORY=1Gi in %s, got: %s", mc.Name, contentsStr)
559+
}
560+
561+
if !strings.Contains(contentsStr, "SYSTEM_RESERVED_CPU=500m") {
562+
t.Errorf("Expected SYSTEM_RESERVED_CPU=500m in %s, got: %s", mc.Name, contentsStr)
563+
}
564+
565+
if !strings.Contains(contentsStr, "SYSTEM_RESERVED_ES=1Gi") {
566+
t.Errorf("Expected SYSTEM_RESERVED_ES=1Gi in %s, got: %s", mc.Name, contentsStr)
567+
}
568+
}
569+
}
570+
}
571+
572+
if !autoSizingFileFound {
573+
t.Errorf("Expected to find %s file in at least one machine config", ctrlcommon.NodeSizingEnabledEnvPath)
574+
}
575+
}

‎templates/arbiter/00-arbiter/_base/files/kubelet-auto-node-sizing-enabled.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
mode: 0644
2+
path: "/etc/node-sizing-enabled.env"
3+
contents:
4+
inline: |
5+
NODE_SIZING_ENABLED=false
6+
SYSTEM_RESERVED_MEMORY=1Gi
7+
SYSTEM_RESERVED_CPU=500m
8+
SYSTEM_RESERVED_ES=1Gi

‎templates/common/_base/files/kubelet-auto-node-sizing-enabled.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ mode: 0644
22
path: "/etc/node-sizing-enabled.env"
33
contents:
44
inline: |
5-
NODE_SIZING_ENABLED=false
5+
NODE_SIZING_ENABLED={{if eq .Infra.Status.ControlPlaneTopology "External"}}false{{else}}true{{end}}
66
SYSTEM_RESERVED_MEMORY=1Gi
77
SYSTEM_RESERVED_CPU=500m
88
SYSTEM_RESERVED_ES=1Gi

‎templates/master/00-master/_base/files/kubelet-auto-node-sizing-enabled.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
mode: 0644
2+
path: "/etc/node-sizing-enabled.env"
3+
contents:
4+
inline: |
5+
NODE_SIZING_ENABLED=false
6+
SYSTEM_RESERVED_MEMORY=1Gi
7+
SYSTEM_RESERVED_CPU=500m
8+
SYSTEM_RESERVED_ES=1Gi

‎test/e2e-2of2/kubeletcfg_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ func TestKubeletConfigMaxPods(t *testing.T) {
6565
kc2 := &mcfgv1.KubeletConfig{
6666
ObjectMeta: metav1.ObjectMeta{Name: "test-200"},
6767
Spec: mcfgv1.KubeletConfigSpec{
68+
AutoSizingReserved: &autoNodeSizing,
6869
KubeletConfig: &runtime.RawExtension{
6970
Raw: kcRaw2,
7071
},

‎test/e2e-bootstrap/bootstrap_test.go‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -389,6 +389,97 @@ metadata:
389389
}
390390
}
391391

392+
func TestNodeSizingEnabled(t *testing.T) {
393+
ctx := context.Background()
394+
395+
testEnv := framework.NewTestEnv(t)
396+
397+
configv1.Install(scheme.Scheme)
398+
configv1alpha1.Install(scheme.Scheme)
399+
mcfgv1.Install(scheme.Scheme)
400+
apioperatorsv1alpha1.Install(scheme.Scheme)
401+
402+
baseTestManifests := loadBaseTestManifests(t)
403+
404+
cfg, err := testEnv.Start()
405+
require.NoError(t, err)
406+
defer func() {
407+
assert.NoError(t, testEnv.Stop())
408+
}()
409+
410+
clientSet := framework.NewClientSetFromConfig(cfg)
411+
412+
_, err = clientSet.Namespaces().Create(ctx, &corev1.Namespace{
413+
ObjectMeta: metav1.ObjectMeta{
414+
Name: framework.OpenshiftConfigNamespace,
415+
},
416+
}, metav1.CreateOptions{})
417+
require.NoError(t, err)
418+
419+
_, err = clientSet.Namespaces().Create(ctx, &corev1.Namespace{
420+
ObjectMeta: metav1.ObjectMeta{
421+
Name: bootstrapTestName,
422+
},
423+
}, metav1.CreateOptions{})
424+
require.NoError(t, err)
425+
426+
objs := append([]runtime.Object{}, baseTestManifests...)
427+
428+
// Add node config
429+
nodeConfigManifest := [][]byte{
430+
[]byte(`apiVersion: config.openshift.io/v1
431+
kind: Node
432+
metadata:
433+
name: cluster`),
434+
}
435+
objs = append(objs, loadRawManifests(t, nodeConfigManifest)...)
436+
437+
fixture := newTestFixture(t, cfg, objs)
438+
defer framework.CleanEnvironment(t, clientSet)
439+
defer fixture.stop()
440+
441+
// Fetch the controller rendered configurations
442+
controllerRenderedMasterConfigName, err := helpers.WaitForRenderedConfigs(t, clientSet, "master", []string{"99-master-ssh", "99-master-generated-registries"}...)
443+
require.NoError(t, err)
444+
t.Logf("Controller rendered master config as %q", controllerRenderedMasterConfigName)
445+
446+
controllerRenderedWorkerConfigName, err := helpers.WaitForRenderedConfigs(t, clientSet, "worker", []string{"99-worker-ssh", "99-worker-generated-registries"}...)
447+
require.NoError(t, err)
448+
t.Logf("Controller rendered worker config as %q", controllerRenderedWorkerConfigName)
449+
450+
// Verify node sizing enabled file for master
451+
verifyNodeSizingEnabled(t, clientSet, controllerRenderedMasterConfigName)
452+
453+
// Verify node sizing enabled file for worker
454+
verifyNodeSizingEnabled(t, clientSet, controllerRenderedWorkerConfigName)
455+
}
456+
457+
func verifyNodeSizingEnabled(t *testing.T, clientSet *framework.ClientSet, renderedConfigName string) {
458+
controllerMC, err := clientSet.MachineConfigs().Get(context.Background(), renderedConfigName, metav1.GetOptions{})
459+
require.NoError(t, err)
460+
461+
ignCfg, err := ctrlcommon.ParseAndConvertConfig(controllerMC.Spec.Config.Raw)
462+
require.NoError(t, err)
463+
464+
// Find the node sizing enabled file
465+
var foundFile bool
466+
for _, file := range ignCfg.Storage.Files {
467+
if file.Path == "/etc/node-sizing-enabled.env" {
468+
foundFile = true
469+
470+
// Decode the file contents
471+
contents, err := ctrlcommon.DecodeIgnitionFileContents(file.Contents.Source, file.Contents.Compression)
472+
require.NoError(t, err, "Failed to decode node-sizing-enabled.env file contents")
473+
474+
contentsStr := string(contents)
475+
require.Contains(t, contentsStr, "NODE_SIZING_ENABLED=true", "Expected /etc/node-sizing-enabled.env to contain NODE_SIZING_ENABLED=true in machine config %s", renderedConfigName)
476+
break
477+
}
478+
}
479+
480+
require.True(t, foundFile, "Expected to find /etc/node-sizing-enabled.env in machine config %s", renderedConfigName)
481+
}
482+
392483
func compareRenderedConfigPool(t *testing.T, clientSet *framework.ClientSet, destDir, poolName, controllerRenderedConfigName string) {
393484
paths, err := filepath.Glob(filepath.Join(destDir, "machine-configs", fmt.Sprintf("rendered-%s-*.yaml", poolName)))
394485
require.NoError(t, err)

0 commit comments

Comments
 (0)