diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index b061438dd..c1e1ef7cc 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -8,6 +8,7 @@ import ( "strings" "syscall" + k0sconfig "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" "github.com/replicatedhq/embedded-cluster/pkg/addons" "github.com/replicatedhq/embedded-cluster/pkg/airgap" @@ -340,8 +341,13 @@ func installAndJoinCluster(ctx context.Context, jcmd *kotsadm.JoinCommandRespons return fmt.Errorf("unable to apply configuration overrides: %w", err) } + profile, err := getFirstDefinedProfile() + if err != nil { + return fmt.Errorf("unable to get first defined profile: %w", err) + } + logrus.Debugf("joining node to cluster") - if err := runK0sInstallCommand(flags.networkInterface, jcmd.K0sJoinCommand); err != nil { + if err := runK0sInstallCommand(flags.networkInterface, jcmd.K0sJoinCommand, profile); err != nil { return fmt.Errorf("unable to join node to cluster: %w", err) } @@ -459,9 +465,24 @@ func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error { return nil } +func getFirstDefinedProfile() (string, error) { + k0scfg, err := os.Open(runtimeconfig.PathToK0sConfig()) + if err != nil { + return "", fmt.Errorf("unable to open k0s config: %w", err) + } + defer k0scfg.Close() + cfg, err := k0sconfig.ConfigFromReader(k0scfg) + if err != nil { + return "", fmt.Errorf("unable to parse k0s config: %w", err) + } + if len(cfg.Spec.WorkerProfiles) > 0 { + return cfg.Spec.WorkerProfiles[0].Name, nil + } + return "", nil +} + // runK0sInstallCommand runs the k0s install command as provided by the kots -// adm api. -func runK0sInstallCommand(networkInterface string, fullcmd string) error { +func runK0sInstallCommand(networkInterface string, fullcmd string, profile string) error { args := strings.Split(fullcmd, " ") args = append(args, "--token-file", "/etc/k0s/join-token") @@ -470,6 +491,10 @@ func runK0sInstallCommand(networkInterface string, fullcmd string) error { return fmt.Errorf("unable to find first valid address: %w", err) } + if profile != "" { + args = append(args, "--profile", profile) + } + args = append(args, config.AdditionalInstallFlags(nodeIP)...) if strings.Contains(fullcmd, "controller") { diff --git a/e2e/install_test.go b/e2e/install_test.go index e49a9b75a..63991e19f 100644 --- a/e2e/install_test.go +++ b/e2e/install_test.go @@ -384,6 +384,11 @@ func TestMultiNodeInstallation(t *testing.T) { if stdout, stderr, err := tc.RunCommandOnNode(0, []string{"single-node-install.sh", "ui", os.Getenv("SHORT_SHA")}); err != nil { t.Fatalf("fail to install embedded-cluster on node 0: %v: %s: %s", err, stdout, stderr) } + t.Logf("checking worker profile on controller node %d", 0) + line := []string{"check-worker-profile.sh"} + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { + t.Fatalf("fail to check worker profile on node %d: %v: %s: %s", 0, err, stdout, stderr) + } if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) @@ -422,6 +427,13 @@ func TestMultiNodeInstallation(t *testing.T) { if stdout, stderr, err := tc.RunCommandOnNode(node, strings.Split(cmd, " ")); err != nil { t.Fatalf("fail to join node %d as a controller: %v: %s: %s", node, err, stdout, stderr) } + + t.Logf("checking worker profile on controller node %d", node) + line := []string{"check-worker-profile.sh"} + if stdout, stderr, err := tc.RunCommandOnNode(node, line); err != nil { + t.Fatalf("fail to check worker profile on node %d: %v: %s: %s", node, err, stdout, stderr) + } + // XXX If we are too aggressive joining nodes we can see the following error being // thrown by kotsadm on its log (and we get a 500 back): // " @@ -436,6 +448,12 @@ func TestMultiNodeInstallation(t *testing.T) { t.Fatalf("fail to join node 3 to the cluster as a worker: %v: %s: %s", err, stdout, stderr) } + t.Logf("checking worker profile on worker node %d", 3) + line = []string{"check-worker-profile.sh"} + if stdout, stderr, err := tc.RunCommandOnNode(3, line); err != nil { + t.Fatalf("fail to check worker profile on node %d: %v: %s: %s", 3, err, stdout, stderr) + } + // wait for the nodes to report as ready. t.Logf("%s: all nodes joined, waiting for them to be ready", time.Now().Format(time.RFC3339)) stdout, stderr, err = tc.RunCommandOnNode(0, []string{"wait-for-ready-nodes.sh", "4"}) @@ -444,7 +462,7 @@ func TestMultiNodeInstallation(t *testing.T) { } t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339)) - line := []string{"check-installation-state.sh", os.Getenv("SHORT_SHA"), k8sVersion()} + line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA"), k8sVersion()} if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr) } @@ -2026,6 +2044,12 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) { if _, _, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err) } + t.Logf("checking worker profile on controller node %d", 0) + line = []string{"check-worker-profile.sh"} + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { + t.Fatalf("fail to check worker profile on node %d: %v: %s: %s", 0, err, stdout, stderr) + } + // remove artifacts after installation to save space line = []string{"rm", "/assets/release.airgap"} if _, _, err := tc.RunCommandOnNode(0, line); err != nil { @@ -2062,6 +2086,11 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) { if stdout, stderr, err := tc.RunCommandOnNode(1, strings.Split(command, " ")); err != nil { t.Fatalf("fail to join node 1 to the cluster as a worker: %v: %s: %s", err, stdout, stderr) } + t.Logf("checking worker profile on worker node %d", 1) + line = []string{"check-worker-profile.sh"} + if stdout, stderr, err := tc.RunCommandOnNode(1, line); err != nil { + t.Fatalf("fail to check worker profile on node %d: %v: %s: %s", 1, err, stdout, stderr) + } // remove the airgap bundle and binary after joining line = []string{"rm", "/assets/release.airgap"} if _, _, err := tc.RunCommandOnNode(1, line); err != nil { @@ -2091,6 +2120,11 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) { if _, _, err := tc.RunCommandOnNode(2, strings.Split(command, " ")); err != nil { t.Fatalf("fail to join node 2 as a controller: %v", err) } + t.Logf("checking worker profile on controller node %d", 2) + line = []string{"check-worker-profile.sh"} + if stdout, stderr, err := tc.RunCommandOnNode(2, line); err != nil { + t.Fatalf("fail to check worker profile on node %d: %v: %s: %s", 2, err, stdout, stderr) + } // remove the airgap bundle and binary after joining line = []string{"rm", "/assets/release.airgap"} if _, _, err := tc.RunCommandOnNode(2, line); err != nil { @@ -2121,6 +2155,11 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) { if _, _, err := tc.RunCommandOnNode(3, line); err != nil { t.Fatalf("fail to join node 3 as a controller in ha mode: %v", err) } + t.Logf("checking worker profile on controller node %d", 3) + line = []string{"check-worker-profile.sh"} + if stdout, stderr, err := tc.RunCommandOnNode(3, line); err != nil { + t.Fatalf("fail to check worker profile on node %d: %v: %s: %s", 3, err, stdout, stderr) + } // remove the airgap bundle and binary after joining line = []string{"rm", "/assets/release.airgap"} if _, _, err := tc.RunCommandOnNode(3, line); err != nil { diff --git a/e2e/kots-release-install-failing-preflights/cluster-config.yaml b/e2e/kots-release-install-failing-preflights/cluster-config.yaml index 4eac429ff..2634327fd 100644 --- a/e2e/kots-release-install-failing-preflights/cluster-config.yaml +++ b/e2e/kots-release-install-failing-preflights/cluster-config.yaml @@ -40,6 +40,11 @@ spec: spec: telemetry: enabled: false + workerProfiles: + - name: ip-forward + values: + allowedUnsafeSysctls: + - net.ipv4.ip_forward extensions: helm: repositories: diff --git a/e2e/kots-release-install-legacydr/cluster-config.yaml b/e2e/kots-release-install-legacydr/cluster-config.yaml index 4eac429ff..2634327fd 100644 --- a/e2e/kots-release-install-legacydr/cluster-config.yaml +++ b/e2e/kots-release-install-legacydr/cluster-config.yaml @@ -40,6 +40,11 @@ spec: spec: telemetry: enabled: false + workerProfiles: + - name: ip-forward + values: + allowedUnsafeSysctls: + - net.ipv4.ip_forward extensions: helm: repositories: diff --git a/e2e/kots-release-install-stable/cluster-config.yaml b/e2e/kots-release-install-stable/cluster-config.yaml index 1e7357761..2b49dfa3e 100644 --- a/e2e/kots-release-install-stable/cluster-config.yaml +++ b/e2e/kots-release-install-stable/cluster-config.yaml @@ -38,6 +38,11 @@ spec: spec: telemetry: enabled: false + workerProfiles: + - name: ip-forward + values: + allowedUnsafeSysctls: + - net.ipv4.ip_forward extensions: helm: repositories: diff --git a/e2e/kots-release-install-warning-preflights/cluster-config.yaml b/e2e/kots-release-install-warning-preflights/cluster-config.yaml index 4eac429ff..2634327fd 100644 --- a/e2e/kots-release-install-warning-preflights/cluster-config.yaml +++ b/e2e/kots-release-install-warning-preflights/cluster-config.yaml @@ -40,6 +40,11 @@ spec: spec: telemetry: enabled: false + workerProfiles: + - name: ip-forward + values: + allowedUnsafeSysctls: + - net.ipv4.ip_forward extensions: helm: repositories: diff --git a/e2e/kots-release-install/cluster-config.yaml b/e2e/kots-release-install/cluster-config.yaml index 6b23e3d65..46abfdacb 100644 --- a/e2e/kots-release-install/cluster-config.yaml +++ b/e2e/kots-release-install/cluster-config.yaml @@ -40,6 +40,11 @@ spec: spec: telemetry: enabled: false + workerProfiles: + - name: ip-forward + values: + allowedUnsafeSysctls: + - net.ipv4.ip_forward extensions: helm: charts: diff --git a/e2e/kots-release-upgrade/cluster-config.yaml b/e2e/kots-release-upgrade/cluster-config.yaml index 46543df5a..6e30d570f 100644 --- a/e2e/kots-release-upgrade/cluster-config.yaml +++ b/e2e/kots-release-upgrade/cluster-config.yaml @@ -46,6 +46,11 @@ spec: spec: telemetry: enabled: false + workerProfiles: + - name: ip-forward + values: + allowedUnsafeSysctls: + - net.ipv4.ip_forward extensions: helm: charts: diff --git a/e2e/scripts/check-installation-state.sh b/e2e/scripts/check-installation-state.sh index cb05a5da6..1937b1711 100755 --- a/e2e/scripts/check-installation-state.sh +++ b/e2e/scripts/check-installation-state.sh @@ -49,7 +49,7 @@ main() { fi # if this is the current version in CI - if echo "$version" | grep -qvE "(pre-minio-removal|1.8.0-k8s)" ; then + if echo "$version" | grep -qvE "(pre-minio-removal|1.8.0-k8s|previous-stable)" ; then validate_data_dirs fi diff --git a/e2e/scripts/check-worker-profile.sh b/e2e/scripts/check-worker-profile.sh new file mode 100755 index 000000000..04f722458 --- /dev/null +++ b/e2e/scripts/check-worker-profile.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euox pipefail + +DIR=/usr/local/bin +. $DIR/common.sh + +main() { + validate_worker_profile +} + +main "$@" diff --git a/e2e/scripts/common.sh b/e2e/scripts/common.sh index 9c4049aff..676554c28 100755 --- a/e2e/scripts/common.sh +++ b/e2e/scripts/common.sh @@ -462,3 +462,21 @@ validate_no_pods_in_crashloop() { exit 1 fi } + +validate_worker_profile() { + # if /etc/systemd/system/k0scontroller.service exists, check it - otherwise check /etc/systemd/system/k0sworker.service + if [ -f /etc/systemd/system/k0scontroller.service ]; then + if ! grep -- "--profile=ip-forward" /etc/systemd/system/k0scontroller.service >/dev/null; then + echo "expected worker profile 'ip-forward' not found in k0scontroller.service" + exit 1 + fi + elif [ -f /etc/systemd/system/k0sworker.service ]; then + if ! grep -- "--profile=ip-forward" /etc/systemd/system/k0sworker.service >/dev/null; then + echo "expected worker profile 'ip-forward' not found in k0sworker.service" + exit 1 + fi + else + echo "expected k0scontroller.service or k0sworker.service not found" + exit 1 + fi +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 75f2853f6..c0d1def42 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -3,6 +3,7 @@ package config import ( "fmt" + "os" "strings" jsonpatch "github.com/evanphx/json-patch" @@ -95,7 +96,7 @@ func PatchK0sConfig(config *k0sconfig.ClusterConfig, patch string) (*k0sconfig.C } // InstallFlags returns a list of default flags to be used when bootstrapping a k0s cluster. -func InstallFlags(nodeIP string) []string { +func InstallFlags(nodeIP string) ([]string, error) { flags := []string{ "install", "controller", @@ -104,9 +105,14 @@ func InstallFlags(nodeIP string) []string { "--no-taints", "-c", runtimeconfig.PathToK0sConfig(), } + profile, err := ProfileInstallFlag() + if err != nil { + return nil, fmt.Errorf("unable to get profile install flag: %w", err) + } + flags = append(flags, profile) flags = append(flags, AdditionalInstallFlags(nodeIP)...) flags = append(flags, AdditionalInstallFlagsController()...) - return flags + return flags, nil } func AdditionalInstallFlags(nodeIP string) []string { @@ -125,6 +131,14 @@ func AdditionalInstallFlagsController() []string { } } +func ProfileInstallFlag() (string, error) { + controllerProfile, err := controllerWorkerProfile() + if err != nil { + return "", fmt.Errorf("unable to get controller worker profile: %w", err) + } + return "--profile=" + controllerProfile, nil +} + // nodeLabels return a slice of string with labels (key=value format) for the node where we // are installing the k0s. func nodeLabels() []string { @@ -163,6 +177,25 @@ func additionalControllerLabels() map[string]string { return map[string]string{} } +func controllerWorkerProfile() (string, error) { + // Read the k0s config file + data, err := os.ReadFile(runtimeconfig.PathToK0sConfig()) + if err != nil { + return "", fmt.Errorf("unable to read k0s config: %w", err) + } + + var cfg k0sconfig.ClusterConfig + if err := k8syaml.Unmarshal(data, &cfg); err != nil { + return "", fmt.Errorf("unable to unmarshal k0s config: %w", err) + } + + // Return the first worker profile name if any exist + if len(cfg.Spec.WorkerProfiles) > 0 { + return cfg.Spec.WorkerProfiles[0].Name, nil + } + return "", nil +} + func AdditionalCharts() []embeddedclusterv1beta1.Chart { clusterConfig := release.GetEmbeddedClusterConfig() if clusterConfig != nil { diff --git a/pkg/helpers/k0s.go b/pkg/helpers/k0s.go index 77b87ecf1..b8a39c408 100644 --- a/pkg/helpers/k0s.go +++ b/pkg/helpers/k0s.go @@ -22,6 +22,30 @@ func K0sClusterConfigTo129Compat(clusterConfig *k0sv1beta1.ClusterConfig) (*unst return nil, fmt.Errorf("convert to unstructured: %w", err) } unst := obj.UnstructuredContent() + + // check the entire spec path before attempting to access "charts" + if unst["spec"] == nil { + return obj, nil + } + if _, ok := unst["spec"].(map[string]interface{}); !ok { + return obj, nil + } + if _, ok := unst["spec"].(map[string]interface{})["extensions"]; !ok { + return obj, nil + } + if _, ok := unst["spec"].(map[string]interface{})["extensions"].(map[string]interface{}); !ok { + return obj, nil + } + if _, ok := unst["spec"].(map[string]interface{})["extensions"].(map[string]interface{})["helm"]; !ok { + return obj, nil + } + if _, ok := unst["spec"].(map[string]interface{})["extensions"].(map[string]interface{})["helm"].(map[string]interface{}); !ok { + return obj, nil + } + if _, ok := unst["spec"].(map[string]interface{})["extensions"].(map[string]interface{})["helm"].(map[string]interface{})["charts"]; !ok { + return obj, nil + } + charts, ok := unst["spec"].(map[string]interface{})["extensions"].(map[string]interface{})["helm"].(map[string]interface{})["charts"].([]interface{}) if !ok { return obj, nil diff --git a/pkg/k0s/install.go b/pkg/k0s/install.go index 286dbf1da..28078bfa6 100644 --- a/pkg/k0s/install.go +++ b/pkg/k0s/install.go @@ -31,7 +31,11 @@ func Install(networkInterface string) error { if err != nil { return fmt.Errorf("unable to find first valid address: %w", err) } - if _, err := helpers.RunCommand(hstbin, config.InstallFlags(nodeIP)...); err != nil { + flags, err := config.InstallFlags(nodeIP) + if err != nil { + return fmt.Errorf("unable to get install flags: %w", err) + } + if _, err := helpers.RunCommand(hstbin, flags...); err != nil { return fmt.Errorf("unable to install: %w", err) } if _, err := helpers.RunCommand(hstbin, "start"); err != nil { @@ -115,11 +119,12 @@ func WriteK0sConfig(ctx context.Context, networkInterface string, airgapBundle s return cfg, nil } -// applyUnsupportedOverrides applies overrides to the k0s configuration. Applies first the -// overrides embedded into the binary and after the ones provided by the user (--overrides). +// applyUnsupportedOverrides applies overrides to the k0s configuration. Applies the +// overrides embedded into the binary and then the ones provided by the user (--overrides). func applyUnsupportedOverrides(cfg *k0sv1beta1.ClusterConfig, overrides string) (*k0sv1beta1.ClusterConfig, error) { embcfg := release.GetEmbeddedClusterConfig() if embcfg != nil { + // Apply vendor k0s overrides overrides := embcfg.Spec.UnsupportedOverrides.K0s var err error cfg, err = config.PatchK0sConfig(cfg, overrides) @@ -132,7 +137,9 @@ func applyUnsupportedOverrides(cfg *k0sv1beta1.ClusterConfig, overrides string) if err != nil { return nil, fmt.Errorf("unable to process overrides file: %w", err) } + if eucfg != nil { + // Apply end user k0s overrides overrides := eucfg.Spec.UnsupportedOverrides.K0s var err error cfg, err = config.PatchK0sConfig(cfg, overrides) @@ -178,6 +185,12 @@ func PatchK0sConfig(path string, patch string) error { } finalcfg.Spec.Storage = result.Spec.Storage } + if result.Spec.WorkerProfiles != nil { + if finalcfg.Spec == nil { + finalcfg.Spec = &k0sv1beta1.ClusterSpec{} + } + finalcfg.Spec.WorkerProfiles = result.Spec.WorkerProfiles + } // This is necessary to install the previous version of k0s in e2e tests // TODO: remove this once the previous version is > 1.29 unstructured, err := helpers.K0sClusterConfigTo129Compat(&finalcfg) diff --git a/pkg/kotsadm/types.go b/pkg/kotsadm/types.go index b5c87244f..4518777a8 100644 --- a/pkg/kotsadm/types.go +++ b/pkg/kotsadm/types.go @@ -21,7 +21,7 @@ type JoinCommandResponse struct { } // extractK0sConfigOverridePatch parses the provided override and returns a dig.Mapping that -// can be then applied on top a k0s configuration file to set both `api` and `storage` spec +// can be then applied on top a k0s configuration file to set `api`, `storage` and `workerProfiles` spec // fields. All other fields in the override are ignored. func (j JoinCommandResponse) extractK0sConfigOverridePatch(data []byte) (dig.Mapping, error) { config := dig.Mapping{} @@ -35,6 +35,10 @@ func (j JoinCommandResponse) extractK0sConfigOverridePatch(data []byte) (dig.Map if storage := config.DigMapping("config", "spec", "storage"); len(storage) > 0 { result.DigMapping("config", "spec")["storage"] = storage } + workerProfiles := config.Dig("config", "spec", "workerProfiles") + if workerProfiles != nil { + result.DigMapping("config", "spec")["workerProfiles"] = workerProfiles + } return result, nil } diff --git a/pkg/runtimeconfig/defaults.go b/pkg/runtimeconfig/defaults.go index 0a9e86918..a671537bb 100644 --- a/pkg/runtimeconfig/defaults.go +++ b/pkg/runtimeconfig/defaults.go @@ -60,7 +60,7 @@ func PathToLog(name string) string { } // K0sBinaryPath returns the path to the k0s binary when it is installed on the node. This -// does not return the binary just after we materilized it but the path we want it to be +// does not return the binary just after we materialized it but the path we want it to be // once it is installed. func K0sBinaryPath() string { return "/usr/local/bin/k0s" diff --git a/tests/dryrun/join_test.go b/tests/dryrun/join_test.go index ed9e8d582..b8eec7e3e 100644 --- a/tests/dryrun/join_test.go +++ b/tests/dryrun/join_test.go @@ -191,7 +191,20 @@ func TestJoinWorkerNode(t *testing.T) { InstallationSpec: ecv1beta1.InstallationSpec{ ClusterID: clusterID.String(), Config: &ecv1beta1.ConfigSpec{ - UnsupportedOverrides: ecv1beta1.UnsupportedOverrides{}, + UnsupportedOverrides: ecv1beta1.UnsupportedOverrides{ + K0s: ` +config: + metadata: + name: foo + spec: + telemetry: + enabled: false + workerProfiles: + - name: ip-forward + values: + allowedUnsafeSysctls: + - net.ipv4.ip_forward`, + }, }, }, }