worker profiles (#1915)

cmd/installer/cli/join.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"syscall"
 
+	k0sconfig "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
 	ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
 	"github.com/replicatedhq/embedded-cluster/pkg/addons"
 	"github.com/replicatedhq/embedded-cluster/pkg/airgap"
@@ -340,8 +341,13 @@ func installAndJoinCluster(ctx context.Context, jcmd *kotsadm.JoinCommandRespons
 		return fmt.Errorf("unable to apply configuration overrides: %w", err)
 	}
 
+	profile, err := getFirstDefinedProfile()
+	if err != nil {
+		return fmt.Errorf("unable to get first defined profile: %w", err)
+	}
+
 	logrus.Debugf("joining node to cluster")
-	if err := runK0sInstallCommand(flags.networkInterface, jcmd.K0sJoinCommand); err != nil {
+	if err := runK0sInstallCommand(flags.networkInterface, jcmd.K0sJoinCommand, profile); err != nil {
 		return fmt.Errorf("unable to join node to cluster: %w", err)
 	}
 
@@ -459,9 +465,24 @@ func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error {
 	return nil
 }
 
+func getFirstDefinedProfile() (string, error) {
+	k0scfg, err := os.Open(runtimeconfig.PathToK0sConfig())
+	if err != nil {
+		return "", fmt.Errorf("unable to open k0s config: %w", err)
+	}
+	defer k0scfg.Close()
+	cfg, err := k0sconfig.ConfigFromReader(k0scfg)
+	if err != nil {
+		return "", fmt.Errorf("unable to parse k0s config: %w", err)
+	}
+	if len(cfg.Spec.WorkerProfiles) > 0 {
+		return cfg.Spec.WorkerProfiles[0].Name, nil
+	}
+	return "", nil
+}
+
 // runK0sInstallCommand runs the k0s install command as provided by the kots
-// adm api.
-func runK0sInstallCommand(networkInterface string, fullcmd string) error {
+func runK0sInstallCommand(networkInterface string, fullcmd string, profile string) error {
 	args := strings.Split(fullcmd, " ")
 	args = append(args, "--token-file", "/etc/k0s/join-token")
 
@@ -470,6 +491,10 @@ func runK0sInstallCommand(networkInterface string, fullcmd string) error {
 		return fmt.Errorf("unable to find first valid address: %w", err)
 	}
 
+	if profile != "" {
+		args = append(args, "--profile", profile)
+	}
+
 	args = append(args, config.AdditionalInstallFlags(nodeIP)...)
 
 	if strings.Contains(fullcmd, "controller") {

e2e/install_test.go

@@ -384,6 +384,11 @@ func TestMultiNodeInstallation(t *testing.T) {
 	if stdout, stderr, err := tc.RunCommandOnNode(0, []string{"single-node-install.sh", "ui", os.Getenv("SHORT_SHA")}); err != nil {
 		t.Fatalf("fail to install embedded-cluster on node 0: %v: %s: %s", err, stdout, stderr)
 	}
+	t.Logf("checking worker profile on controller node %d", 0)
+	line := []string{"check-worker-profile.sh"}
+	if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
+		t.Fatalf("fail to check worker profile on node %d: %v: %s: %s", 0, err, stdout, stderr)
+	}
 
 	if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil {
 		t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr)
@@ -422,6 +427,13 @@ func TestMultiNodeInstallation(t *testing.T) {
 		if stdout, stderr, err := tc.RunCommandOnNode(node, strings.Split(cmd, " ")); err != nil {
 			t.Fatalf("fail to join node %d as a controller: %v: %s: %s", node, err, stdout, stderr)
 		}
+
+		t.Logf("checking worker profile on controller node %d", node)
+		line := []string{"check-worker-profile.sh"}
+		if stdout, stderr, err := tc.RunCommandOnNode(node, line); err != nil {
+			t.Fatalf("fail to check worker profile on node %d: %v: %s: %s", node, err, stdout, stderr)
+		}
+
 		// XXX If we are too aggressive joining nodes we can see the following error being
 		// thrown by kotsadm on its log (and we get a 500 back):
 		// "
@@ -436,6 +448,12 @@ func TestMultiNodeInstallation(t *testing.T) {
 		t.Fatalf("fail to join node 3 to the cluster as a worker: %v: %s: %s", err, stdout, stderr)
 	}
 
+	t.Logf("checking worker profile on worker node %d", 3)
+	line = []string{"check-worker-profile.sh"}
+	if stdout, stderr, err := tc.RunCommandOnNode(3, line); err != nil {
+		t.Fatalf("fail to check worker profile on node %d: %v: %s: %s", 3, err, stdout, stderr)
+	}
+
 	// wait for the nodes to report as ready.
 	t.Logf("%s: all nodes joined, waiting for them to be ready", time.Now().Format(time.RFC3339))
 	stdout, stderr, err = tc.RunCommandOnNode(0, []string{"wait-for-ready-nodes.sh", "4"})
@@ -444,7 +462,7 @@ func TestMultiNodeInstallation(t *testing.T) {
 	}
 
 	t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
-	line := []string{"check-installation-state.sh", os.Getenv("SHORT_SHA"), k8sVersion()}
+	line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA"), k8sVersion()}
 	if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
 		t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr)
 	}
@@ -2026,6 +2044,12 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) {
 	if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
 		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
 	}
+	t.Logf("checking worker profile on controller node %d", 0)
+	line = []string{"check-worker-profile.sh"}
+	if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
+		t.Fatalf("fail to check worker profile on node %d: %v: %s: %s", 0, err, stdout, stderr)
+	}
+
 	// remove artifacts after installation to save space
 	line = []string{"rm", "/assets/release.airgap"}
 	if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
@@ -2062,6 +2086,11 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) {
 	if stdout, stderr, err := tc.RunCommandOnNode(1, strings.Split(command, " ")); err != nil {
 		t.Fatalf("fail to join node 1 to the cluster as a worker: %v: %s: %s", err, stdout, stderr)
 	}
+	t.Logf("checking worker profile on worker node %d", 1)
+	line = []string{"check-worker-profile.sh"}
+	if stdout, stderr, err := tc.RunCommandOnNode(1, line); err != nil {
+		t.Fatalf("fail to check worker profile on node %d: %v: %s: %s", 1, err, stdout, stderr)
+	}
 	// remove the airgap bundle and binary after joining
 	line = []string{"rm", "/assets/release.airgap"}
 	if _, _, err := tc.RunCommandOnNode(1, line); err != nil {
@@ -2091,6 +2120,11 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) {
 	if _, _, err := tc.RunCommandOnNode(2, strings.Split(command, " ")); err != nil {
 		t.Fatalf("fail to join node 2 as a controller: %v", err)
 	}
+	t.Logf("checking worker profile on controller node %d", 2)
+	line = []string{"check-worker-profile.sh"}
+	if stdout, stderr, err := tc.RunCommandOnNode(2, line); err != nil {
+		t.Fatalf("fail to check worker profile on node %d: %v: %s: %s", 2, err, stdout, stderr)
+	}
 	// remove the airgap bundle and binary after joining
 	line = []string{"rm", "/assets/release.airgap"}
 	if _, _, err := tc.RunCommandOnNode(2, line); err != nil {
@@ -2121,6 +2155,11 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) {
 	if _, _, err := tc.RunCommandOnNode(3, line); err != nil {
 		t.Fatalf("fail to join node 3 as a controller in ha mode: %v", err)
 	}
+	t.Logf("checking worker profile on controller node %d", 3)
+	line = []string{"check-worker-profile.sh"}
+	if stdout, stderr, err := tc.RunCommandOnNode(3, line); err != nil {
+		t.Fatalf("fail to check worker profile on node %d: %v: %s: %s", 3, err, stdout, stderr)
+	}
 	// remove the airgap bundle and binary after joining
 	line = []string{"rm", "/assets/release.airgap"}
 	if _, _, err := tc.RunCommandOnNode(3, line); err != nil {

e2e/kots-release-install-failing-preflights/cluster-config.yaml

@@ -40,6 +40,11 @@ spec:
         spec:
           telemetry:
             enabled: false
+          workerProfiles:
+          - name: ip-forward
+            values:
+              allowedUnsafeSysctls:
+              - net.ipv4.ip_forward
   extensions:
     helm:
       repositories:

e2e/kots-release-install-legacydr/cluster-config.yaml

@@ -40,6 +40,11 @@ spec:
         spec:
           telemetry:
             enabled: false
+          workerProfiles:
+          - name: ip-forward
+            values:
+              allowedUnsafeSysctls:
+              - net.ipv4.ip_forward
   extensions:
     helm:
       repositories:

e2e/kots-release-install-stable/cluster-config.yaml

@@ -38,6 +38,11 @@ spec:
         spec:
           telemetry:
             enabled: false
+          workerProfiles:
+          - name: ip-forward
+            values:
+              allowedUnsafeSysctls:
+              - net.ipv4.ip_forward
   extensions:
     helm:
       repositories:

e2e/kots-release-install-warning-preflights/cluster-config.yaml

@@ -40,6 +40,11 @@ spec:
         spec:
           telemetry:
             enabled: false
+          workerProfiles:
+          - name: ip-forward
+            values:
+              allowedUnsafeSysctls:
+              - net.ipv4.ip_forward
   extensions:
     helm:
       repositories:

e2e/kots-release-install/cluster-config.yaml

@@ -40,6 +40,11 @@ spec:
         spec:
           telemetry:
             enabled: false
+          workerProfiles:
+          - name: ip-forward
+            values:
+              allowedUnsafeSysctls:
+              - net.ipv4.ip_forward
   extensions:
     helm:
       charts:

e2e/kots-release-upgrade/cluster-config.yaml

@@ -46,6 +46,11 @@ spec:
         spec:
           telemetry:
             enabled: false
+          workerProfiles:
+          - name: ip-forward
+            values:
+              allowedUnsafeSysctls:
+              - net.ipv4.ip_forward
   extensions:
     helm:
       charts:

e2e/scripts/check-installation-state.sh

@@ -49,7 +49,7 @@ main() {
     fi
 
     # if this is the current version in CI
-    if echo "$version" | grep -qvE "(pre-minio-removal|1.8.0-k8s)" ; then
+    if echo "$version" | grep -qvE "(pre-minio-removal|1.8.0-k8s|previous-stable)" ; then
         validate_data_dirs
     fi
 

e2e/scripts/check-worker-profile.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euox pipefail
+
+DIR=/usr/local/bin
+. $DIR/common.sh
+
+main() {
+    validate_worker_profile
+}
+
+main "$@"

e2e/scripts/common.sh

@@ -462,3 +462,21 @@ validate_no_pods_in_crashloop() {
         exit 1
     fi
 }
+
+validate_worker_profile() {
+    # if /etc/systemd/system/k0scontroller.service exists, check it - otherwise check /etc/systemd/system/k0sworker.service
+    if [ -f /etc/systemd/system/k0scontroller.service ]; then
+        if ! grep -- "--profile=ip-forward" /etc/systemd/system/k0scontroller.service >/dev/null; then
+            echo "expected worker profile 'ip-forward' not found in k0scontroller.service"
+            exit 1
+        fi
+    elif [ -f /etc/systemd/system/k0sworker.service ]; then
+        if ! grep -- "--profile=ip-forward" /etc/systemd/system/k0sworker.service >/dev/null; then
+            echo "expected worker profile 'ip-forward' not found in k0sworker.service"
+            exit 1
+        fi
+    else
+        echo "expected k0scontroller.service or k0sworker.service not found"
+        exit 1
+    fi
+}

pkg/config/config.go

@@ -3,6 +3,7 @@ package config
 
 import (
 	"fmt"
+	"os"
 	"strings"
 
 	jsonpatch "github.com/evanphx/json-patch"
@@ -95,7 +96,7 @@ func PatchK0sConfig(config *k0sconfig.ClusterConfig, patch string) (*k0sconfig.C
 }
 
 // InstallFlags returns a list of default flags to be used when bootstrapping a k0s cluster.
-func InstallFlags(nodeIP string) []string {
+func InstallFlags(nodeIP string) ([]string, error) {
 	flags := []string{
 		"install",
 		"controller",
@@ -104,9 +105,14 @@ func InstallFlags(nodeIP string) []string {
 		"--no-taints",
 		"-c", runtimeconfig.PathToK0sConfig(),
 	}
+	profile, err := ProfileInstallFlag()
+	if err != nil {
+		return nil, fmt.Errorf("unable to get profile install flag: %w", err)
+	}
+	flags = append(flags, profile)
 	flags = append(flags, AdditionalInstallFlags(nodeIP)...)
 	flags = append(flags, AdditionalInstallFlagsController()...)
-	return flags
+	return flags, nil
 }
 
 func AdditionalInstallFlags(nodeIP string) []string {
@@ -125,6 +131,14 @@ func AdditionalInstallFlagsController() []string {
 	}
 }
 
+func ProfileInstallFlag() (string, error) {
+	controllerProfile, err := controllerWorkerProfile()
+	if err != nil {
+		return "", fmt.Errorf("unable to get controller worker profile: %w", err)
+	}
+	return "--profile=" + controllerProfile, nil
+}
+
 // nodeLabels return a slice of string with labels (key=value format) for the node where we
 // are installing the k0s.
 func nodeLabels() []string {
@@ -163,6 +177,25 @@ func additionalControllerLabels() map[string]string {
 	return map[string]string{}
 }
 
+func controllerWorkerProfile() (string, error) {
+	// Read the k0s config file
+	data, err := os.ReadFile(runtimeconfig.PathToK0sConfig())
+	if err != nil {
+		return "", fmt.Errorf("unable to read k0s config: %w", err)
+	}
+
+	var cfg k0sconfig.ClusterConfig
+	if err := k8syaml.Unmarshal(data, &cfg); err != nil {
+		return "", fmt.Errorf("unable to unmarshal k0s config: %w", err)
+	}
+
+	// Return the first worker profile name if any exist
+	if len(cfg.Spec.WorkerProfiles) > 0 {
+		return cfg.Spec.WorkerProfiles[0].Name, nil
+	}
+	return "", nil
+}
+
 func AdditionalCharts() []embeddedclusterv1beta1.Chart {
 	clusterConfig := release.GetEmbeddedClusterConfig()
 	if clusterConfig != nil {

pkg/helpers/k0s.go

@@ -22,6 +22,30 @@ func K0sClusterConfigTo129Compat(clusterConfig *k0sv1beta1.ClusterConfig) (*unst
 		return nil, fmt.Errorf("convert to unstructured: %w", err)
 	}
 	unst := obj.UnstructuredContent()
+
+	// check the entire spec path before attempting to access "charts"
+	if unst["spec"] == nil {
+		return obj, nil
+	}
+	if _, ok := unst["spec"].(map[string]interface{}); !ok {
+		return obj, nil
+	}
+	if _, ok := unst["spec"].(map[string]interface{})["extensions"]; !ok {
+		return obj, nil
+	}
+	if _, ok := unst["spec"].(map[string]interface{})["extensions"].(map[string]interface{}); !ok {
+		return obj, nil
+	}
+	if _, ok := unst["spec"].(map[string]interface{})["extensions"].(map[string]interface{})["helm"]; !ok {
+		return obj, nil
+	}
+	if _, ok := unst["spec"].(map[string]interface{})["extensions"].(map[string]interface{})["helm"].(map[string]interface{}); !ok {
+		return obj, nil
+	}
+	if _, ok := unst["spec"].(map[string]interface{})["extensions"].(map[string]interface{})["helm"].(map[string]interface{})["charts"]; !ok {
+		return obj, nil
+	}
+
 	charts, ok := unst["spec"].(map[string]interface{})["extensions"].(map[string]interface{})["helm"].(map[string]interface{})["charts"].([]interface{})
 	if !ok {
 		return obj, nil