From 63a6a04eef0cec79ebfc3b9c1334b781f0b4e0a3 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Mon, 17 Mar 2025 07:26:29 -0500 Subject: [PATCH 01/48] worker profiles investigation (#1915) * always enable last defined worker profile * handle errors * unmarshal to clusterconfig * rename var * extract k0s patch from unsupportedOverrides * oops * try using patch function * debug * pass overrides through to install flags * simplify worker profile determination code * just read config from disk * choose worker profile based on node role * fix crds * log profiles * f * add worker profiles to spec * workerprofiles in config * generate * split override functions * include worker profiles in cfg on join * move to unsupported overrides * always apply first defined worker profile if any are set * regenerate static * manifests --------- Co-authored-by: hedge-sparrow --- cmd/installer/cli/install.go | 2 +- cmd/installer/cli/join.go | 24 ++++++++++ e2e/kots-release-install/cluster-config.yaml | 6 +++ kinds/apis/v1beta1/config_types.go | 3 ++ kinds/apis/v1beta1/zz_generated.deepcopy.go | 7 +++ .../charts/crds/templates/resources.yaml | 32 +++++++++++++ ...mbeddedcluster.replicated.com_configs.yaml | 17 +++++++ ...dcluster.replicated.com_installations.yaml | 17 +++++++ .../config-embeddedcluster-v1beta1.json | 43 +++++++++++++++-- pkg/config/config.go | 37 +++++++++++++- pkg/k0s/install.go | 48 +++++++++++++++++-- pkg/runtimeconfig/defaults.go | 2 +- 12 files changed, 227 insertions(+), 11 deletions(-) diff --git a/cmd/installer/cli/install.go b/cmd/installer/cli/install.go index 840b50c74..e7cd9e090 100644 --- a/cmd/installer/cli/install.go +++ b/cmd/installer/cli/install.go @@ -632,7 +632,7 @@ func installAndStartCluster(ctx context.Context, networkInterface string, airgap } logrus.Debugf("installing k0s") - if err := k0s.Install(networkInterface); err != nil { + if err := k0s.Install(networkInterface, cfg); err != nil { return nil, fmt.Errorf("install cluster: %w", err) } diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index c0e624acd..165c872ea 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -8,6 +8,7 @@ import ( "strings" "syscall" + k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" "github.com/replicatedhq/embedded-cluster/pkg/addons" "github.com/replicatedhq/embedded-cluster/pkg/airgap" @@ -325,6 +326,11 @@ func installAndJoinCluster(ctx context.Context, jcmd *kotsadm.JoinCommandRespons return fmt.Errorf("unable to apply network configuration: %w", err) } + logrus.Debugf("applying worker profiles") + if err := applyWorkerProfiles(jcmd); err != nil { + return fmt.Errorf("unable to apply worker profiles: %w", err) + } + logrus.Debugf("applying configuration overrides") if err := applyJoinConfigurationOverrides(jcmd); err != nil { return fmt.Errorf("unable to apply configuration overrides: %w", err) @@ -417,6 +423,24 @@ func startAndWaitForK0s(ctx context.Context, name string, jcmd *kotsadm.JoinComm return nil } +func applyWorkerProfiles(jcmd *kotsadm.JoinCommandResponse) error { + if jcmd.InstallationSpec.Config != nil { + cfgProfiles := jcmd.InstallationSpec.Config.UnsupportedOverrides.WorkerProfiles + if len(cfgProfiles) > 0 { + cfg := k0sv1beta1.ClusterConfig{} + cfg.Spec.WorkerProfiles = cfgProfiles + data, err := k8syaml.Marshal(cfg) + if err != nil { + return fmt.Errorf("unable to marshal cluster config: %w", err) + } + if err := k0s.PatchK0sConfig(runtimeconfig.PathToK0sConfig(), string(data)); err != nil { + return fmt.Errorf("unable to patch config with embedded data: %w", err) + } + } + } + return nil +} + // applyJoinConfigurationOverrides applies both config overrides received from the kots api. // Applies first the EmbeddedOverrides and then the EndUserOverrides. func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error { diff --git a/e2e/kots-release-install/cluster-config.yaml b/e2e/kots-release-install/cluster-config.yaml index 01ddaceff..7c0c0e3c4 100644 --- a/e2e/kots-release-install/cluster-config.yaml +++ b/e2e/kots-release-install/cluster-config.yaml @@ -11,6 +11,7 @@ spec: labels: controller-label: controller-label-value name: controller-test + workerProfile: ip-forward custom: - labels: abc-test-label: abc-test-label-value @@ -19,6 +20,11 @@ spec: - labels: xyz-test-label: xyz-value name: xyz + workerProfiles: + - name: ip-forward + values: + allowedUnsafeSysctls: + - net.ipv4.ip_forward unsupportedOverrides: builtInExtensions: - name: admin-console diff --git a/kinds/apis/v1beta1/config_types.go b/kinds/apis/v1beta1/config_types.go index 924c5f30d..6760ddc57 100644 --- a/kinds/apis/v1beta1/config_types.go +++ b/kinds/apis/v1beta1/config_types.go @@ -35,6 +35,9 @@ type UnsupportedOverrides struct { // BuiltInExtensions holds overrides for the default add-ons we ship // with Embedded Cluster. BuiltInExtensions []BuiltInExtension `json:"builtInExtensions,omitempty"` + // WorkerProfiles holds the worker profiles used to configure the cluster. + // The profile named "default" will be applied by default. + WorkerProfiles []k0sv1beta1.WorkerProfile `json:"workerProfiles,omitempty"` } // BuiltInExtension holds the override for a built-in extension (add-on). diff --git a/kinds/apis/v1beta1/zz_generated.deepcopy.go b/kinds/apis/v1beta1/zz_generated.deepcopy.go index 5b8892738..c8af91551 100644 --- a/kinds/apis/v1beta1/zz_generated.deepcopy.go +++ b/kinds/apis/v1beta1/zz_generated.deepcopy.go @@ -621,6 +621,13 @@ func (in *UnsupportedOverrides) DeepCopyInto(out *UnsupportedOverrides) { *out = make([]BuiltInExtension, len(*in)) copy(*out, *in) } + if in.WorkerProfiles != nil { + in, out := &in.WorkerProfiles, &out.WorkerProfiles + *out = make([]k0sv1beta1.WorkerProfile, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UnsupportedOverrides. diff --git a/operator/charts/embedded-cluster-operator/charts/crds/templates/resources.yaml b/operator/charts/embedded-cluster-operator/charts/crds/templates/resources.yaml index 122e3ad7f..da3f9454b 100644 --- a/operator/charts/embedded-cluster-operator/charts/crds/templates/resources.yaml +++ b/operator/charts/embedded-cluster-operator/charts/crds/templates/resources.yaml @@ -217,6 +217,22 @@ spec: layout inside this configuration is very dynamic we have chosen to use a string here. type: string + workerProfiles: + description: |- + WorkerProfiles holds the worker profiles used to configure the cluster. + The profile named "default" will be applied by default. + items: + description: WorkerProfile worker profile + properties: + name: + description: String; name to use as profile selector for the worker process + type: string + values: + description: Worker Mapping object + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: array type: object v2Enabled: description: |- @@ -514,6 +530,22 @@ spec: layout inside this configuration is very dynamic we have chosen to use a string here. type: string + workerProfiles: + description: |- + WorkerProfiles holds the worker profiles used to configure the cluster. + The profile named "default" will be applied by default. + items: + description: WorkerProfile worker profile + properties: + name: + description: String; name to use as profile selector for the worker process + type: string + values: + description: Worker Mapping object + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: array type: object v2Enabled: description: |- diff --git a/operator/config/crd/bases/embeddedcluster.replicated.com_configs.yaml b/operator/config/crd/bases/embeddedcluster.replicated.com_configs.yaml index e94921efc..84539aec0 100644 --- a/operator/config/crd/bases/embeddedcluster.replicated.com_configs.yaml +++ b/operator/config/crd/bases/embeddedcluster.replicated.com_configs.yaml @@ -226,6 +226,23 @@ spec: layout inside this configuration is very dynamic we have chosen to use a string here. type: string + workerProfiles: + description: |- + WorkerProfiles holds the worker profiles used to configure the cluster. + The profile named "default" will be applied by default. + items: + description: WorkerProfile worker profile + properties: + name: + description: String; name to use as profile selector for + the worker process + type: string + values: + description: Worker Mapping object + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: array type: object v2Enabled: description: |- diff --git a/operator/config/crd/bases/embeddedcluster.replicated.com_installations.yaml b/operator/config/crd/bases/embeddedcluster.replicated.com_installations.yaml index 4f8b96a4a..e74540b31 100644 --- a/operator/config/crd/bases/embeddedcluster.replicated.com_installations.yaml +++ b/operator/config/crd/bases/embeddedcluster.replicated.com_installations.yaml @@ -288,6 +288,23 @@ spec: layout inside this configuration is very dynamic we have chosen to use a string here. type: string + workerProfiles: + description: |- + WorkerProfiles holds the worker profiles used to configure the cluster. + The profile named "default" will be applied by default. + items: + description: WorkerProfile worker profile + properties: + name: + description: String; name to use as profile selector + for the worker process + type: string + values: + description: Worker Mapping object + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: array type: object v2Enabled: description: |- diff --git a/operator/schemas/config-embeddedcluster-v1beta1.json b/operator/schemas/config-embeddedcluster-v1beta1.json index ba4b90bdc..5aeebba44 100644 --- a/operator/schemas/config-embeddedcluster-v1beta1.json +++ b/operator/schemas/config-embeddedcluster-v1beta1.json @@ -50,9 +50,9 @@ "type": "integer" }, "timeout": { - "description": "Timeout specifies the timeout for how long to wait for the chart installation to finish.", - "type": "integer", - "format": "int64" + "description": "Timeout specifies the timeout for how long to wait for the chart installation to finish.\nA duration string is a sequence of decimal numbers, each with optional fraction and a unit suffix, such as \"300ms\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\".", + "type": "string", + "x-kubernetes-int-or-string": true }, "values": { "type": "string" @@ -69,30 +69,39 @@ "repositories": { "type": "array", "items": { + "description": "Repository describes single repository entry. Fields map to the CLI flags for the \"helm add\" command", "type": "object", "properties": { "caFile": { + "description": "CA bundle file to use when verifying HTTPS-enabled servers.", "type": "string" }, "certFile": { + "description": "The TLS certificate file to use for HTTPS client authentication.", "type": "string" }, "insecure": { + "description": "Whether to skip TLS certificate checks when connecting to the repository.", "type": "boolean" }, "keyfile": { + "description": "The TLS key file to use for HTTPS client authentication.", "type": "string" }, "name": { + "description": "The repository name.", "type": "string" }, "password": { + "description": "Password for Basic HTTP authentication.", "type": "string" }, "url": { + "description": "The repository URL.", "type": "string" }, "username": { + "description": "Username for Basic HTTP authentication.", "type": "string" } } @@ -151,6 +160,9 @@ } } } + }, + "workerProfile": { + "type": "string" } } }, @@ -198,6 +210,9 @@ } } } + }, + "workerProfile": { + "type": "string" } } } @@ -236,8 +251,30 @@ } } }, + "v2Enabled": { + "description": "V2Enabled is a temporary property that can be used to opt-in to the new installer. If set,\nin addition to using the new v2 install method, v1 installations will be migrated to v2 on\nupgrade. This property will be removed once the new installer is fully implemented and the\nold installer is removed.", + "type": "boolean" + }, "version": { "type": "string" + }, + "workerProfiles": { + "type": "array", + "items": { + "description": "WorkerProfile worker profile", + "type": "object", + "properties": { + "name": { + "description": "String; name to use as profile selector for the worker process", + "type": "string" + }, + "values": { + "description": "Worker Mapping object", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + } + } + } } } }, diff --git a/pkg/config/config.go b/pkg/config/config.go index 504366ae5..cb1a44764 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -93,7 +93,7 @@ func PatchK0sConfig(config *k0sconfig.ClusterConfig, patch string) (*k0sconfig.C } // InstallFlags returns a list of default flags to be used when bootstrapping a k0s cluster. -func InstallFlags(nodeIP string) []string { +func InstallFlags(nodeIP string, cfg *k0sconfig.ClusterConfig) ([]string, error) { flags := []string{ "install", "controller", @@ -102,9 +102,14 @@ func InstallFlags(nodeIP string) []string { "--no-taints", "-c", runtimeconfig.PathToK0sConfig(), } + profile, err := ProfileInstallFlag(cfg) + if err != nil { + return nil, fmt.Errorf("unable to get profile install flag: %w", err) + } + flags = append(flags, profile) flags = append(flags, AdditionalInstallFlags(nodeIP)...) flags = append(flags, AdditionalInstallFlagsController()...) - return flags + return flags, nil } func AdditionalInstallFlags(nodeIP string) []string { @@ -123,6 +128,22 @@ func AdditionalInstallFlagsController() []string { } } +func ProfileInstallFlag(cfg *k0sconfig.ClusterConfig) (string, error) { + controllerProfile := controllerWorkerProfile() + if controllerProfile == "" { + return "", nil + } + + // make sure that the controller profile role name exists in the worker profiles + for _, profile := range cfg.Spec.WorkerProfiles { + if profile.Name == controllerProfile { + return "--profile=" + controllerProfile, nil + } + } + + return "", fmt.Errorf("controller profile %q not found in k0s worker profiles", controllerProfile) +} + // nodeLabels return a slice of string with labels (key=value format) for the node where we // are installing the k0s. func nodeLabels() []string { @@ -165,6 +186,18 @@ func additionalControllerLabels() map[string]string { return map[string]string{} } +func controllerWorkerProfile() string { + clusterConfig, err := release.GetEmbeddedClusterConfig() + if err == nil { + if clusterConfig != nil { + if len(clusterConfig.Spec.UnsupportedOverrides.WorkerProfiles) > 0 { + return clusterConfig.Spec.UnsupportedOverrides.WorkerProfiles[0].Name + } + } + } + return "" +} + func AdditionalCharts() []embeddedclusterv1beta1.Chart { clusterConfig, err := release.GetEmbeddedClusterConfig() if err == nil { diff --git a/pkg/k0s/install.go b/pkg/k0s/install.go index e69df19f7..827b01e20 100644 --- a/pkg/k0s/install.go +++ b/pkg/k0s/install.go @@ -19,7 +19,7 @@ import ( // Install runs the k0s install command and waits for it to finish. If no configuration // is found one is generated. -func Install(networkInterface string) error { +func Install(networkInterface string, cfg *k0sv1beta1.ClusterConfig) error { ourbin := runtimeconfig.PathToEmbeddedClusterBinary("k0s") hstbin := runtimeconfig.K0sBinaryPath() if err := helpers.MoveFile(ourbin, hstbin); err != nil { @@ -30,7 +30,11 @@ func Install(networkInterface string) error { if err != nil { return fmt.Errorf("unable to find first valid address: %w", err) } - if _, err := helpers.RunCommand(hstbin, config.InstallFlags(nodeIP)...); err != nil { + flags, err := config.InstallFlags(nodeIP, cfg) + if err != nil { + return fmt.Errorf("unable to get install flags: %w", err) + } + if _, err := helpers.RunCommand(hstbin, flags...); err != nil { return fmt.Errorf("unable to install: %w", err) } if _, err := helpers.RunCommand(hstbin, "start"); err != nil { @@ -81,6 +85,11 @@ func WriteK0sConfig(ctx context.Context, networkInterface string, airgapBundle s } } + cfg, err = applyWorkerProfiles(cfg, overrides) + if err != nil { + return nil, fmt.Errorf("unable to apply worker profiles: %w", err) + } + cfg, err = applyUnsupportedOverrides(cfg, overrides) if err != nil { return nil, fmt.Errorf("unable to apply unsupported overrides: %w", err) @@ -107,14 +116,43 @@ func WriteK0sConfig(ctx context.Context, networkInterface string, airgapBundle s return cfg, nil } -// applyUnsupportedOverrides applies overrides to the k0s configuration. Applies first the -// overrides embedded into the binary and after the ones provided by the user (--overrides). +// applyWorkerProfiles applies worker profiles to the k0s configuration. Applies the +// worker profiles embedded into the binary and then the ones provided by the user +// (--overrides). +func applyWorkerProfiles(cfg *k0sv1beta1.ClusterConfig, overrides string) (*k0sv1beta1.ClusterConfig, error) { + embcfg, err := release.GetEmbeddedClusterConfig() + if err != nil { + return nil, fmt.Errorf("unable to get embedded cluster config: %w", err) + } + + if embcfg != nil && len(embcfg.Spec.UnsupportedOverrides.WorkerProfiles) > 0 { + // Apply vendor WorkerProfiles + cfg.Spec.WorkerProfiles = embcfg.Spec.UnsupportedOverrides.WorkerProfiles + } + + eucfg, err := helpers.ParseEndUserConfig(overrides) + if err != nil { + return nil, fmt.Errorf("unable to process overrides file: %w", err) + } + + if eucfg != nil && len(eucfg.Spec.UnsupportedOverrides.WorkerProfiles) > 0 { + // Apply end user WorkerProfiles (these take priority over vendor profiles) + cfg.Spec.WorkerProfiles = eucfg.Spec.UnsupportedOverrides.WorkerProfiles + } + + return cfg, nil +} + +// applyUnsupportedOverrides applies overrides to the k0s configuration. Applies the +// overrides embedded into the binary and then the ones provided by the user (--overrides). func applyUnsupportedOverrides(cfg *k0sv1beta1.ClusterConfig, overrides string) (*k0sv1beta1.ClusterConfig, error) { embcfg, err := release.GetEmbeddedClusterConfig() if err != nil { return nil, fmt.Errorf("unable to get embedded cluster config: %w", err) } + if embcfg != nil { + // Apply vendor k0s overrides overrides := embcfg.Spec.UnsupportedOverrides.K0s cfg, err = config.PatchK0sConfig(cfg, overrides) if err != nil { @@ -126,7 +164,9 @@ func applyUnsupportedOverrides(cfg *k0sv1beta1.ClusterConfig, overrides string) if err != nil { return nil, fmt.Errorf("unable to process overrides file: %w", err) } + if eucfg != nil { + // Apply end user k0s overrides overrides := eucfg.Spec.UnsupportedOverrides.K0s cfg, err = config.PatchK0sConfig(cfg, overrides) if err != nil { diff --git a/pkg/runtimeconfig/defaults.go b/pkg/runtimeconfig/defaults.go index 560984708..fe26430d5 100644 --- a/pkg/runtimeconfig/defaults.go +++ b/pkg/runtimeconfig/defaults.go @@ -49,7 +49,7 @@ func PathToLog(name string) string { } // K0sBinaryPath returns the path to the k0s binary when it is installed on the node. This -// does not return the binary just after we materilized it but the path we want it to be +// does not return the binary just after we materialized it but the path we want it to be // once it is installed. func K0sBinaryPath() string { return "/usr/local/bin/k0s" From e4afc3ea34b9b588924d74280e78e424b4b85035 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Mon, 17 Mar 2025 12:57:37 +0000 Subject: [PATCH 02/48] schemas --- .../config-embeddedcluster-v1beta1.json | 43 ++++++++----------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/operator/schemas/config-embeddedcluster-v1beta1.json b/operator/schemas/config-embeddedcluster-v1beta1.json index 5aeebba44..1c8cc555c 100644 --- a/operator/schemas/config-embeddedcluster-v1beta1.json +++ b/operator/schemas/config-embeddedcluster-v1beta1.json @@ -160,9 +160,6 @@ } } } - }, - "workerProfile": { - "type": "string" } } }, @@ -210,9 +207,6 @@ } } } - }, - "workerProfile": { - "type": "string" } } } @@ -248,6 +242,25 @@ "k0s": { "description": "K0s holds the overrides used to configure k0s. These overrides\nare merged on top of the default k0s configuration. As the data\nlayout inside this configuration is very dynamic we have chosen\nto use a string here.", "type": "string" + }, + "workerProfiles": { + "description": "WorkerProfiles holds the worker profiles used to configure the cluster.\nThe profile named \"default\" will be applied by default.", + "type": "array", + "items": { + "description": "WorkerProfile worker profile", + "type": "object", + "properties": { + "name": { + "description": "String; name to use as profile selector for the worker process", + "type": "string" + }, + "values": { + "description": "Worker Mapping object", + "type": "object", + "x-kubernetes-preserve-unknown-fields": true + } + } + } } } }, @@ -257,24 +270,6 @@ }, "version": { "type": "string" - }, - "workerProfiles": { - "type": "array", - "items": { - "description": "WorkerProfile worker profile", - "type": "object", - "properties": { - "name": { - "description": "String; name to use as profile selector for the worker process", - "type": "string" - }, - "values": { - "description": "Worker Mapping object", - "type": "object", - "x-kubernetes-preserve-unknown-fields": true - } - } - } } } }, From 4c7540c9fc14b70664ceceaf963de0ad86b01e40 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Mon, 17 Mar 2025 13:16:36 +0000 Subject: [PATCH 03/48] schemas --- operator/schemas/config-embeddedcluster-v1beta1.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/operator/schemas/config-embeddedcluster-v1beta1.json b/operator/schemas/config-embeddedcluster-v1beta1.json index 578035b45..48f8bb61f 100644 --- a/operator/schemas/config-embeddedcluster-v1beta1.json +++ b/operator/schemas/config-embeddedcluster-v1beta1.json @@ -278,10 +278,6 @@ } } }, - "v2Enabled": { - "description": "V2Enabled is a temporary property that can be used to opt-in to the new installer. If set,\nin addition to using the new v2 install method, v1 installations will be migrated to v2 on\nupgrade. This property will be removed once the new installer is fully implemented and the\nold installer is removed.", - "type": "boolean" - }, "version": { "type": "string" } From f2f88bfe34a43d44f8de837478c104cf3efa0722 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Wed, 19 Mar 2025 11:05:08 +0000 Subject: [PATCH 04/48] try and get first defined profile from kots join command response --- cmd/installer/cli/join.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index 165c872ea..18664e219 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -337,7 +337,7 @@ func installAndJoinCluster(ctx context.Context, jcmd *kotsadm.JoinCommandRespons } logrus.Debugf("joining node to cluster") - if err := runK0sInstallCommand(flags.networkInterface, jcmd.K0sJoinCommand); err != nil { + if err := runK0sInstallCommand(flags.networkInterface, jcmd.K0sJoinCommand, getFirstDefinedProfileFlag(jcmd)); err != nil { return fmt.Errorf("unable to join node to cluster: %w", err) } @@ -472,12 +472,28 @@ func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error { return nil } +// getFirstDefinedProfileFlag returns the name of the first defined worker profile +// from the returned join command config +func getFirstDefinedProfileFlag(jcmd *kotsadm.JoinCommandResponse) string { + if jcmd.InstallationSpec.Config != nil { + cfgProfiles := jcmd.InstallationSpec.Config.UnsupportedOverrides.WorkerProfiles + if len(cfgProfiles) > 0 { + return cfgProfiles[len(cfgProfiles)-1].Name + } + } + return "" +} + // runK0sInstallCommand runs the k0s install command as provided by the kots // adm api. -func runK0sInstallCommand(networkInterface string, fullcmd string) error { +func runK0sInstallCommand(networkInterface string, fullcmd string, profile string) error { args := strings.Split(fullcmd, " ") args = append(args, "--token-file", "/etc/k0s/join-token") + if profile != "" { + args = append(args, "--profile="+profile) + } + nodeIP, err := netutils.FirstValidAddress(networkInterface) if err != nil { return fmt.Errorf("unable to find first valid address: %w", err) From 0b8062705a08217ec85d12aa4d4f7b5271df6e02 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Wed, 19 Mar 2025 13:01:32 +0000 Subject: [PATCH 05/48] fix outdated release.GetEmbeddedClusterConfig calls --- pkg/config/config.go | 10 ++++------ pkg/k0s/install.go | 6 +----- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index f1bd60e90..5724e35d2 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -183,12 +183,10 @@ func additionalControllerLabels() map[string]string { } func controllerWorkerProfile() string { - clusterConfig, err := release.GetEmbeddedClusterConfig() - if err == nil { - if clusterConfig != nil { - if len(clusterConfig.Spec.UnsupportedOverrides.WorkerProfiles) > 0 { - return clusterConfig.Spec.UnsupportedOverrides.WorkerProfiles[0].Name - } + clusterConfig := release.GetEmbeddedClusterConfig() + if clusterConfig != nil { + if len(clusterConfig.Spec.UnsupportedOverrides.WorkerProfiles) > 0 { + return clusterConfig.Spec.UnsupportedOverrides.WorkerProfiles[0].Name } } return "" diff --git a/pkg/k0s/install.go b/pkg/k0s/install.go index 19df1637d..4187057f1 100644 --- a/pkg/k0s/install.go +++ b/pkg/k0s/install.go @@ -128,11 +128,7 @@ func WriteK0sConfig(ctx context.Context, networkInterface string, airgapBundle s // worker profiles embedded into the binary and then the ones provided by the user // (--overrides). func applyWorkerProfiles(cfg *k0sv1beta1.ClusterConfig, overrides string) (*k0sv1beta1.ClusterConfig, error) { - embcfg, err := release.GetEmbeddedClusterConfig() - if err != nil { - return nil, fmt.Errorf("unable to get embedded cluster config: %w", err) - } - + embcfg := release.GetEmbeddedClusterConfig() if embcfg != nil && len(embcfg.Spec.UnsupportedOverrides.WorkerProfiles) > 0 { // Apply vendor WorkerProfiles cfg.Spec.WorkerProfiles = embcfg.Spec.UnsupportedOverrides.WorkerProfiles From 6035ebb85377e8b59547d4904d0f53ffbf2d3b42 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Wed, 19 Mar 2025 13:47:05 +0000 Subject: [PATCH 06/48] remove unneeded check for profile name --- pkg/config/config.go | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 5724e35d2..fe62cf9b2 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -102,11 +102,7 @@ func InstallFlags(nodeIP string, cfg *k0sconfig.ClusterConfig) ([]string, error) "--no-taints", "-c", runtimeconfig.PathToK0sConfig(), } - profile, err := ProfileInstallFlag(cfg) - if err != nil { - return nil, fmt.Errorf("unable to get profile install flag: %w", err) - } - flags = append(flags, profile) + flags = append(flags, ProfileInstallFlag(cfg)) flags = append(flags, AdditionalInstallFlags(nodeIP)...) flags = append(flags, AdditionalInstallFlagsController()...) return flags, nil @@ -128,20 +124,12 @@ func AdditionalInstallFlagsController() []string { } } -func ProfileInstallFlag(cfg *k0sconfig.ClusterConfig) (string, error) { +func ProfileInstallFlag(cfg *k0sconfig.ClusterConfig) string { controllerProfile := controllerWorkerProfile() if controllerProfile == "" { - return "", nil - } - - // make sure that the controller profile role name exists in the worker profiles - for _, profile := range cfg.Spec.WorkerProfiles { - if profile.Name == controllerProfile { - return "--profile=" + controllerProfile, nil - } + return "" } - - return "", fmt.Errorf("controller profile %q not found in k0s worker profiles", controllerProfile) + return "--profile=" + controllerProfile } // nodeLabels return a slice of string with labels (key=value format) for the node where we From 014ea09e3ab5e582acd942e99441f639527aa12e Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Wed, 19 Mar 2025 14:11:12 +0000 Subject: [PATCH 07/48] change workerprofiles position in test config --- e2e/kots-release-install/cluster-config.yaml | 11 +++++----- .../cluster-config.yaml | 10 ++++----- .../testdata/override-setting-ip-forward.yaml | 22 ++++++++++--------- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/e2e/kots-release-install/cluster-config.yaml b/e2e/kots-release-install/cluster-config.yaml index d5b13ddb7..82a5fe7a5 100644 --- a/e2e/kots-release-install/cluster-config.yaml +++ b/e2e/kots-release-install/cluster-config.yaml @@ -14,7 +14,6 @@ spec: labels: controller-label: controller-label-value name: controller-test - workerProfile: ip-forward custom: - labels: abc-test-label: abc-test-label-value @@ -23,12 +22,12 @@ spec: - labels: xyz-test-label: xyz-value name: xyz - workerProfiles: - - name: ip-forward - values: - allowedUnsafeSysctls: - - net.ipv4.ip_forward unsupportedOverrides: + workerProfiles: + - name: ip-forward + values: + allowedUnsafeSysctls: + - net.ipv4.ip_forward builtInExtensions: - name: admin-console values: | diff --git a/e2e/kots-release-unsupported-overrides/cluster-config.yaml b/e2e/kots-release-unsupported-overrides/cluster-config.yaml index 62132cec4..cfc5636d9 100644 --- a/e2e/kots-release-unsupported-overrides/cluster-config.yaml +++ b/e2e/kots-release-unsupported-overrides/cluster-config.yaml @@ -10,6 +10,11 @@ spec: replicatedAppDomain: "ec-e2e-replicated-app.testcluster.net" proxyRegistryDomain: "ec-e2e-proxy.testcluster.net" unsupportedOverrides: + workerProfiles: + - name: ip-forward + values: + allowedUnsafeSysctls: + - net.ipv4.ip_forward builtInExtensions: - name: admin-console values: | @@ -27,8 +32,3 @@ spec: spec: telemetry: enabled: true - workerProfiles: - - name: ip-forward - values: - allowedUnsafeSysctls: - - net.ipv4.ip_forward diff --git a/pkg/config/testdata/override-setting-ip-forward.yaml b/pkg/config/testdata/override-setting-ip-forward.yaml index ea9d654b1..3bfd1b40e 100644 --- a/pkg/config/testdata/override-setting-ip-forward.yaml +++ b/pkg/config/testdata/override-setting-ip-forward.yaml @@ -14,11 +14,12 @@ config: |- override: |- config: spec: - workerProfiles: - - name: ip-forward - values: - allowedUnsafeSysctls: - - net.ipv4.ip_forward + unsupportedOverrides + workerProfiles: + - name: ip-forward + values: + allowedUnsafeSysctls: + - net.ipv4.ip_forward expected: |- apiVersion: k0s.k0sproject.io/v1beta1 kind: ClusterConfig @@ -32,8 +33,9 @@ expected: |- provider: calico telemetry: enabled: false - workerProfiles: - - name: ip-forward - values: - allowedUnsafeSysctls: - - net.ipv4.ip_forward + unsupportedOverrides + workerProfiles: + - name: ip-forward + values: + allowedUnsafeSysctls: + - net.ipv4.ip_forward From 1041615dfcb3729e97bff04033b451ff1a0ab92e Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Wed, 19 Mar 2025 15:25:25 +0000 Subject: [PATCH 08/48] reverse selection of join profile --- cmd/installer/cli/join.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index 4a358aaac..96e01009a 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -479,7 +479,7 @@ func getFirstDefinedProfileFlag(jcmd *kotsadm.JoinCommandResponse) string { if jcmd.InstallationSpec.Config != nil { cfgProfiles := jcmd.InstallationSpec.Config.UnsupportedOverrides.WorkerProfiles if len(cfgProfiles) > 0 { - return cfgProfiles[len(cfgProfiles)-1].Name + return cfgProfiles[0].Name } } return "" From 28e35a207094d9a1dea3c9fd93f5bc76256f16c0 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Wed, 19 Mar 2025 16:40:15 +0000 Subject: [PATCH 09/48] kots image override --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index dd521b2ad..55de75ef8 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ ARCH ?= $(shell go env GOARCH) APP_NAME = embedded-cluster ADMIN_CONSOLE_CHART_REPO_OVERRIDE = -ADMIN_CONSOLE_IMAGE_OVERRIDE = +ADMIN_CONSOLE_IMAGE_OVERRIDE = ttl.sh/ash/kotsadm:24h ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = K0S_VERSION = v1.30.9+k0s.0 From daadf0328374d17cd3c498e219f957d2a2330101 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Thu, 20 Mar 2025 09:16:41 +0000 Subject: [PATCH 10/48] test print spec --- cmd/installer/cli/join.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index 96e01009a..eaab00225 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -476,6 +476,7 @@ func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error { // getFirstDefinedProfileFlag returns the name of the first defined worker profile // from the returned join command config func getFirstDefinedProfileFlag(jcmd *kotsadm.JoinCommandResponse) string { + fmt.Printf("%+v\n", jcmd.InstallationSpec) if jcmd.InstallationSpec.Config != nil { cfgProfiles := jcmd.InstallationSpec.Config.UnsupportedOverrides.WorkerProfiles if len(cfgProfiles) > 0 { From 7d23c91b04dc9876490cd4f3d61b26fd32012b54 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Thu, 20 Mar 2025 09:42:36 +0000 Subject: [PATCH 11/48] test print spec --- cmd/installer/cli/join.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index eaab00225..8c712419e 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -476,8 +476,10 @@ func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error { // getFirstDefinedProfileFlag returns the name of the first defined worker profile // from the returned join command config func getFirstDefinedProfileFlag(jcmd *kotsadm.JoinCommandResponse) string { - fmt.Printf("%+v\n", jcmd.InstallationSpec) if jcmd.InstallationSpec.Config != nil { + fmt.Printf("%+v\n", jcmd.InstallationSpec.Config) + fmt.Printf("%+v\n", jcmd.InstallationSpec.Config.UnsupportedOverrides) + fmt.Printf("%+v\n", jcmd.InstallationSpec.Config.UnsupportedOverrides.WorkerProfiles) cfgProfiles := jcmd.InstallationSpec.Config.UnsupportedOverrides.WorkerProfiles if len(cfgProfiles) > 0 { return cfgProfiles[0].Name From b24198db80b31397bf01c5aae3081f0d87364188 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Thu, 20 Mar 2025 10:43:57 +0000 Subject: [PATCH 12/48] move worker profiles back to k0s override config --- cmd/installer/cli/join.go | 27 ++++++++++++++++----------- pkg/config/config.go | 20 +++++++++++++++----- pkg/k0s/install.go | 28 ---------------------------- 3 files changed, 31 insertions(+), 44 deletions(-) diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index 8c712419e..ba4406c35 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -425,11 +425,15 @@ func startAndWaitForK0s(ctx context.Context, name string, jcmd *kotsadm.JoinComm } func applyWorkerProfiles(jcmd *kotsadm.JoinCommandResponse) error { - if jcmd.InstallationSpec.Config != nil { - cfgProfiles := jcmd.InstallationSpec.Config.UnsupportedOverrides.WorkerProfiles - if len(cfgProfiles) > 0 { + if jcmd.InstallationSpec.Config != nil && jcmd.InstallationSpec.Config.UnsupportedOverrides.K0s != "" { + var k0sConfig k0sv1beta1.ClusterConfig + if err := yaml.Unmarshal([]byte(jcmd.InstallationSpec.Config.UnsupportedOverrides.K0s), &k0sConfig); err != nil { + logrus.Debugf("unable to parse k0s config: %v", err) + return nil + } + if len(k0sConfig.Spec.WorkerProfiles) > 0 { cfg := k0sv1beta1.ClusterConfig{} - cfg.Spec.WorkerProfiles = cfgProfiles + cfg.Spec.WorkerProfiles = k0sConfig.Spec.WorkerProfiles data, err := k8syaml.Marshal(cfg) if err != nil { return fmt.Errorf("unable to marshal cluster config: %w", err) @@ -476,13 +480,14 @@ func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error { // getFirstDefinedProfileFlag returns the name of the first defined worker profile // from the returned join command config func getFirstDefinedProfileFlag(jcmd *kotsadm.JoinCommandResponse) string { - if jcmd.InstallationSpec.Config != nil { - fmt.Printf("%+v\n", jcmd.InstallationSpec.Config) - fmt.Printf("%+v\n", jcmd.InstallationSpec.Config.UnsupportedOverrides) - fmt.Printf("%+v\n", jcmd.InstallationSpec.Config.UnsupportedOverrides.WorkerProfiles) - cfgProfiles := jcmd.InstallationSpec.Config.UnsupportedOverrides.WorkerProfiles - if len(cfgProfiles) > 0 { - return cfgProfiles[0].Name + if jcmd.InstallationSpec.Config != nil && jcmd.InstallationSpec.Config.UnsupportedOverrides.K0s != "" { + var k0sConfig k0sv1beta1.ClusterConfig + if err := yaml.Unmarshal([]byte(jcmd.InstallationSpec.Config.UnsupportedOverrides.K0s), &k0sConfig); err != nil { + logrus.Debugf("unable to parse k0s config: %v", err) + return "" + } + if len(k0sConfig.Spec.WorkerProfiles) > 0 { + return k0sConfig.Spec.WorkerProfiles[0].Name } } return "" diff --git a/pkg/config/config.go b/pkg/config/config.go index fe62cf9b2..d14265af8 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -3,6 +3,7 @@ package config import ( "fmt" + "os" "strings" jsonpatch "github.com/evanphx/json-patch" @@ -171,11 +172,20 @@ func additionalControllerLabels() map[string]string { } func controllerWorkerProfile() string { - clusterConfig := release.GetEmbeddedClusterConfig() - if clusterConfig != nil { - if len(clusterConfig.Spec.UnsupportedOverrides.WorkerProfiles) > 0 { - return clusterConfig.Spec.UnsupportedOverrides.WorkerProfiles[0].Name - } + // Read the k0s config file + data, err := os.ReadFile(runtimeconfig.PathToK0sConfig()) + if err != nil { + return "" + } + + var cfg k0sconfig.ClusterConfig + if err := k8syaml.Unmarshal(data, &cfg); err != nil { + return "" + } + + // Return the first worker profile name if any exist + if len(cfg.Spec.WorkerProfiles) > 0 { + return cfg.Spec.WorkerProfiles[0].Name } return "" } diff --git a/pkg/k0s/install.go b/pkg/k0s/install.go index 4187057f1..3c70bc271 100644 --- a/pkg/k0s/install.go +++ b/pkg/k0s/install.go @@ -93,11 +93,6 @@ func WriteK0sConfig(ctx context.Context, networkInterface string, airgapBundle s } } - cfg, err = applyWorkerProfiles(cfg, overrides) - if err != nil { - return nil, fmt.Errorf("unable to apply worker profiles: %w", err) - } - cfg, err = applyUnsupportedOverrides(cfg, overrides) if err != nil { return nil, fmt.Errorf("unable to apply unsupported overrides: %w", err) @@ -124,29 +119,6 @@ func WriteK0sConfig(ctx context.Context, networkInterface string, airgapBundle s return cfg, nil } -// applyWorkerProfiles applies worker profiles to the k0s configuration. Applies the -// worker profiles embedded into the binary and then the ones provided by the user -// (--overrides). -func applyWorkerProfiles(cfg *k0sv1beta1.ClusterConfig, overrides string) (*k0sv1beta1.ClusterConfig, error) { - embcfg := release.GetEmbeddedClusterConfig() - if embcfg != nil && len(embcfg.Spec.UnsupportedOverrides.WorkerProfiles) > 0 { - // Apply vendor WorkerProfiles - cfg.Spec.WorkerProfiles = embcfg.Spec.UnsupportedOverrides.WorkerProfiles - } - - eucfg, err := helpers.ParseEndUserConfig(overrides) - if err != nil { - return nil, fmt.Errorf("unable to process overrides file: %w", err) - } - - if eucfg != nil && len(eucfg.Spec.UnsupportedOverrides.WorkerProfiles) > 0 { - // Apply end user WorkerProfiles (these take priority over vendor profiles) - cfg.Spec.WorkerProfiles = eucfg.Spec.UnsupportedOverrides.WorkerProfiles - } - - return cfg, nil -} - // applyUnsupportedOverrides applies overrides to the k0s configuration. Applies the // overrides embedded into the binary and then the ones provided by the user (--overrides). func applyUnsupportedOverrides(cfg *k0sv1beta1.ClusterConfig, overrides string) (*k0sv1beta1.ClusterConfig, error) { From b7768bea164979a85b6b7b9101ed8aa3892ba1bb Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Thu, 20 Mar 2025 10:50:12 +0000 Subject: [PATCH 13/48] no custom kots --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 55de75ef8..dd521b2ad 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ ARCH ?= $(shell go env GOARCH) APP_NAME = embedded-cluster ADMIN_CONSOLE_CHART_REPO_OVERRIDE = -ADMIN_CONSOLE_IMAGE_OVERRIDE = ttl.sh/ash/kotsadm:24h +ADMIN_CONSOLE_IMAGE_OVERRIDE = ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE = ADMIN_CONSOLE_KURL_PROXY_IMAGE_OVERRIDE = K0S_VERSION = v1.30.9+k0s.0 From 4af0dfed01256e34b06027c77abf0840b8ff54c0 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Thu, 20 Mar 2025 11:06:18 +0000 Subject: [PATCH 14/48] move test config worker profiles --- e2e/kots-release-install/cluster-config.yaml | 10 +++++----- .../cluster-config.yaml | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/e2e/kots-release-install/cluster-config.yaml b/e2e/kots-release-install/cluster-config.yaml index 82a5fe7a5..cd137375f 100644 --- a/e2e/kots-release-install/cluster-config.yaml +++ b/e2e/kots-release-install/cluster-config.yaml @@ -23,11 +23,6 @@ spec: xyz-test-label: xyz-value name: xyz unsupportedOverrides: - workerProfiles: - - name: ip-forward - values: - allowedUnsafeSysctls: - - net.ipv4.ip_forward builtInExtensions: - name: admin-console values: | @@ -45,6 +40,11 @@ spec: spec: telemetry: enabled: false + workerProfiles: + - name: ip-forward + values: + allowedUnsafeSysctls: + - net.ipv4.ip_forward extensions: helm: charts: diff --git a/e2e/kots-release-unsupported-overrides/cluster-config.yaml b/e2e/kots-release-unsupported-overrides/cluster-config.yaml index cfc5636d9..62132cec4 100644 --- a/e2e/kots-release-unsupported-overrides/cluster-config.yaml +++ b/e2e/kots-release-unsupported-overrides/cluster-config.yaml @@ -10,11 +10,6 @@ spec: replicatedAppDomain: "ec-e2e-replicated-app.testcluster.net" proxyRegistryDomain: "ec-e2e-proxy.testcluster.net" unsupportedOverrides: - workerProfiles: - - name: ip-forward - values: - allowedUnsafeSysctls: - - net.ipv4.ip_forward builtInExtensions: - name: admin-console values: | @@ -32,3 +27,8 @@ spec: spec: telemetry: enabled: true + workerProfiles: + - name: ip-forward + values: + allowedUnsafeSysctls: + - net.ipv4.ip_forward From eb68feed157d1257b9215a028e2110df8f262dd1 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Thu, 20 Mar 2025 14:05:30 +0000 Subject: [PATCH 15/48] don't care about profiles key here --- cmd/installer/cli/join.go | 43 +++---------------- .../testdata/override-setting-ip-forward.yaml | 22 +++++----- 2 files changed, 17 insertions(+), 48 deletions(-) diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index ba4406c35..089dfbc62 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -8,7 +8,6 @@ import ( "strings" "syscall" - k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" "github.com/replicatedhq/embedded-cluster/pkg/addons" "github.com/replicatedhq/embedded-cluster/pkg/airgap" @@ -326,11 +325,6 @@ func installAndJoinCluster(ctx context.Context, jcmd *kotsadm.JoinCommandRespons return fmt.Errorf("unable to apply network configuration: %w", err) } - logrus.Debugf("applying worker profiles") - if err := applyWorkerProfiles(jcmd); err != nil { - return fmt.Errorf("unable to apply worker profiles: %w", err) - } - logrus.Debugf("applying configuration overrides") if err := applyJoinConfigurationOverrides(jcmd); err != nil { return fmt.Errorf("unable to apply configuration overrides: %w", err) @@ -424,28 +418,6 @@ func startAndWaitForK0s(ctx context.Context, name string, jcmd *kotsadm.JoinComm return nil } -func applyWorkerProfiles(jcmd *kotsadm.JoinCommandResponse) error { - if jcmd.InstallationSpec.Config != nil && jcmd.InstallationSpec.Config.UnsupportedOverrides.K0s != "" { - var k0sConfig k0sv1beta1.ClusterConfig - if err := yaml.Unmarshal([]byte(jcmd.InstallationSpec.Config.UnsupportedOverrides.K0s), &k0sConfig); err != nil { - logrus.Debugf("unable to parse k0s config: %v", err) - return nil - } - if len(k0sConfig.Spec.WorkerProfiles) > 0 { - cfg := k0sv1beta1.ClusterConfig{} - cfg.Spec.WorkerProfiles = k0sConfig.Spec.WorkerProfiles - data, err := k8syaml.Marshal(cfg) - if err != nil { - return fmt.Errorf("unable to marshal cluster config: %w", err) - } - if err := k0s.PatchK0sConfig(runtimeconfig.PathToK0sConfig(), string(data)); err != nil { - return fmt.Errorf("unable to patch config with embedded data: %w", err) - } - } - } - return nil -} - // applyJoinConfigurationOverrides applies both config overrides received from the kots api. // Applies first the EmbeddedOverrides and then the EndUserOverrides. func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error { @@ -480,14 +452,13 @@ func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error { // getFirstDefinedProfileFlag returns the name of the first defined worker profile // from the returned join command config func getFirstDefinedProfileFlag(jcmd *kotsadm.JoinCommandResponse) string { - if jcmd.InstallationSpec.Config != nil && jcmd.InstallationSpec.Config.UnsupportedOverrides.K0s != "" { - var k0sConfig k0sv1beta1.ClusterConfig - if err := yaml.Unmarshal([]byte(jcmd.InstallationSpec.Config.UnsupportedOverrides.K0s), &k0sConfig); err != nil { - logrus.Debugf("unable to parse k0s config: %v", err) - return "" - } - if len(k0sConfig.Spec.WorkerProfiles) > 0 { - return k0sConfig.Spec.WorkerProfiles[0].Name + if jcmd.InstallationSpec.Config != nil { + fmt.Printf("%+v\n", jcmd.InstallationSpec.Config) + fmt.Printf("%+v\n", jcmd.InstallationSpec.Config.UnsupportedOverrides) + fmt.Printf("%+v\n", jcmd.InstallationSpec.Config.UnsupportedOverrides.WorkerProfiles) + cfgProfiles := jcmd.InstallationSpec.Config.UnsupportedOverrides.WorkerProfiles + if len(cfgProfiles) > 0 { + return cfgProfiles[0].Name } } return "" diff --git a/pkg/config/testdata/override-setting-ip-forward.yaml b/pkg/config/testdata/override-setting-ip-forward.yaml index 3bfd1b40e..ea9d654b1 100644 --- a/pkg/config/testdata/override-setting-ip-forward.yaml +++ b/pkg/config/testdata/override-setting-ip-forward.yaml @@ -14,12 +14,11 @@ config: |- override: |- config: spec: - unsupportedOverrides - workerProfiles: - - name: ip-forward - values: - allowedUnsafeSysctls: - - net.ipv4.ip_forward + workerProfiles: + - name: ip-forward + values: + allowedUnsafeSysctls: + - net.ipv4.ip_forward expected: |- apiVersion: k0s.k0sproject.io/v1beta1 kind: ClusterConfig @@ -33,9 +32,8 @@ expected: |- provider: calico telemetry: enabled: false - unsupportedOverrides - workerProfiles: - - name: ip-forward - values: - allowedUnsafeSysctls: - - net.ipv4.ip_forward + workerProfiles: + - name: ip-forward + values: + allowedUnsafeSysctls: + - net.ipv4.ip_forward From 8c2c25c2e7325d095d390b7c43fe71ca0229bbf8 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Thu, 20 Mar 2025 14:42:56 +0000 Subject: [PATCH 16/48] oops --- cmd/installer/cli/join.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index 089dfbc62..787516176 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -8,6 +8,7 @@ import ( "strings" "syscall" + k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" "github.com/replicatedhq/embedded-cluster/pkg/addons" "github.com/replicatedhq/embedded-cluster/pkg/airgap" @@ -452,13 +453,14 @@ func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error { // getFirstDefinedProfileFlag returns the name of the first defined worker profile // from the returned join command config func getFirstDefinedProfileFlag(jcmd *kotsadm.JoinCommandResponse) string { - if jcmd.InstallationSpec.Config != nil { - fmt.Printf("%+v\n", jcmd.InstallationSpec.Config) - fmt.Printf("%+v\n", jcmd.InstallationSpec.Config.UnsupportedOverrides) - fmt.Printf("%+v\n", jcmd.InstallationSpec.Config.UnsupportedOverrides.WorkerProfiles) - cfgProfiles := jcmd.InstallationSpec.Config.UnsupportedOverrides.WorkerProfiles - if len(cfgProfiles) > 0 { - return cfgProfiles[0].Name + if jcmd.InstallationSpec.Config != nil && jcmd.InstallationSpec.Config.UnsupportedOverrides.K0s != "" { + var k0sConfig k0sv1beta1.ClusterConfig + if err := yaml.Unmarshal([]byte(jcmd.InstallationSpec.Config.UnsupportedOverrides.K0s), &k0sConfig); err != nil { + logrus.Debugf("unable to parse k0s config: %v", err) + return "" + } + if len(k0sConfig.Spec.WorkerProfiles) > 0 { + return k0sConfig.Spec.WorkerProfiles[0].Name } } return "" From 8ae806fe0547fb075a91716a8074404b8212abfe Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Thu, 20 Mar 2025 15:10:27 +0000 Subject: [PATCH 17/48] more safety --- cmd/installer/cli/join.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index 787516176..c11ec11d4 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -453,17 +453,21 @@ func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error { // getFirstDefinedProfileFlag returns the name of the first defined worker profile // from the returned join command config func getFirstDefinedProfileFlag(jcmd *kotsadm.JoinCommandResponse) string { - if jcmd.InstallationSpec.Config != nil && jcmd.InstallationSpec.Config.UnsupportedOverrides.K0s != "" { - var k0sConfig k0sv1beta1.ClusterConfig - if err := yaml.Unmarshal([]byte(jcmd.InstallationSpec.Config.UnsupportedOverrides.K0s), &k0sConfig); err != nil { - logrus.Debugf("unable to parse k0s config: %v", err) - return "" - } - if len(k0sConfig.Spec.WorkerProfiles) > 0 { - return k0sConfig.Spec.WorkerProfiles[0].Name - } + if jcmd == nil || jcmd.InstallationSpec.Config == nil { + return "" + } + if jcmd.InstallationSpec.Config.UnsupportedOverrides.K0s == "" { + return "" + } + var k0scfg k0sv1beta1.ClusterConfig + if err := yaml.Unmarshal([]byte(jcmd.InstallationSpec.Config.UnsupportedOverrides.K0s), &k0scfg); err != nil { + logrus.Debugf("unable to parse k0s config: %v", err) + return "" + } + if k0scfg.Spec == nil || len(k0scfg.Spec.WorkerProfiles) == 0 { + return "" } - return "" + return k0scfg.Spec.WorkerProfiles[0].Name } // runK0sInstallCommand runs the k0s install command as provided by the kots From b35da65281d159a82808c19fe83d5f855cf85a10 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Thu, 20 Mar 2025 15:14:27 +0000 Subject: [PATCH 18/48] print debug --- cmd/installer/cli/join.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index c11ec11d4..f3b48bcaf 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -464,6 +464,7 @@ func getFirstDefinedProfileFlag(jcmd *kotsadm.JoinCommandResponse) string { logrus.Debugf("unable to parse k0s config: %v", err) return "" } + fmt.Printf("%+v\n", k0scfg) if k0scfg.Spec == nil || len(k0scfg.Spec.WorkerProfiles) == 0 { return "" } From e1f7ce15b755d3c2bab4da702f724a573f759b2c Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Thu, 20 Mar 2025 15:59:17 +0000 Subject: [PATCH 19/48] extract k0s config from installation object properly --- cmd/installer/cli/join.go | 8 +++++++- pkg/config/config.go | 6 +++--- pkg/config/config_test.go | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index f3b48bcaf..39d972ed9 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -459,8 +459,14 @@ func getFirstDefinedProfileFlag(jcmd *kotsadm.JoinCommandResponse) string { if jcmd.InstallationSpec.Config.UnsupportedOverrides.K0s == "" { return "" } + + cfgString, err := config.ExtractK0sConfigPatch(jcmd.InstallationSpec.Config.UnsupportedOverrides.K0s) + if err != nil { + logrus.Debugf("unable to extract k0s config from installation object: %v", err) + return "" + } var k0scfg k0sv1beta1.ClusterConfig - if err := yaml.Unmarshal([]byte(jcmd.InstallationSpec.Config.UnsupportedOverrides.K0s), &k0scfg); err != nil { + if err := yaml.Unmarshal([]byte(cfgString), &k0scfg); err != nil { logrus.Debugf("unable to parse k0s config: %v", err) return "" } diff --git a/pkg/config/config.go b/pkg/config/config.go index d14265af8..a5d6b496b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -39,8 +39,8 @@ func RenderK0sConfig(proxyRegistryDomain string) *k0sconfig.ClusterConfig { return cfg } -// extractK0sConfigPatch extracts the k0s config portion of the provided patch. -func extractK0sConfigPatch(raw string) (string, error) { +// ExtractK0sConfigPatch extracts the k0s config portion of the provided patch. +func ExtractK0sConfigPatch(raw string) (string, error) { type PatchBody struct { Config map[string]interface{} `yaml:"config"` } @@ -62,7 +62,7 @@ func PatchK0sConfig(config *k0sconfig.ClusterConfig, patch string) (*k0sconfig.C if patch == "" { return config, nil } - patch, err := extractK0sConfigPatch(patch) + patch, err := ExtractK0sConfigPatch(patch) if err != nil { return nil, fmt.Errorf("unable to extract k0s config patch: %w", err) } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index ed38cdbf0..7b6e7cc22 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -78,7 +78,7 @@ func Test_extractK0sConfigPatch(t *testing.T) { t.Run(tname, func(t *testing.T) { req := require.New(t) - extracted, err := extractK0sConfigPatch(tt.Override) + extracted, err := ExtractK0sConfigPatch(tt.Override) req.NoError(err) var actual map[string]interface{} From 6c79ac97ed518965c339132c13094cdf31818801 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Thu, 20 Mar 2025 16:33:01 +0000 Subject: [PATCH 20/48] uh --- cmd/installer/cli/join.go | 4 ++-- kinds/apis/v1beta1/config_types.go | 3 --- pkg/config/config.go | 4 ++-- pkg/k0s/install.go | 2 +- pkg/kotsadm/types.go | 3 +++ 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index 39d972ed9..32a249bce 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -470,7 +470,7 @@ func getFirstDefinedProfileFlag(jcmd *kotsadm.JoinCommandResponse) string { logrus.Debugf("unable to parse k0s config: %v", err) return "" } - fmt.Printf("%+v\n", k0scfg) + fmt.Printf("%+v\n", k0scfg.Spec) if k0scfg.Spec == nil || len(k0scfg.Spec.WorkerProfiles) == 0 { return "" } @@ -484,7 +484,7 @@ func runK0sInstallCommand(networkInterface string, fullcmd string, profile strin args = append(args, "--token-file", "/etc/k0s/join-token") if profile != "" { - args = append(args, "--profile="+profile) + args = append(args, "--profile", profile) } nodeIP, err := netutils.FirstValidAddress(networkInterface) diff --git a/kinds/apis/v1beta1/config_types.go b/kinds/apis/v1beta1/config_types.go index 2cbcd8ed2..1f75f072b 100644 --- a/kinds/apis/v1beta1/config_types.go +++ b/kinds/apis/v1beta1/config_types.go @@ -35,9 +35,6 @@ type UnsupportedOverrides struct { // BuiltInExtensions holds overrides for the default add-ons we ship // with Embedded Cluster. BuiltInExtensions []BuiltInExtension `json:"builtInExtensions,omitempty"` - // WorkerProfiles holds the worker profiles used to configure the cluster. - // The profile named "default" will be applied by default. - WorkerProfiles []k0sv1beta1.WorkerProfile `json:"workerProfiles,omitempty"` } // BuiltInExtension holds the override for a built-in extension (add-on). diff --git a/pkg/config/config.go b/pkg/config/config.go index a5d6b496b..604ea78c8 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -94,7 +94,7 @@ func PatchK0sConfig(config *k0sconfig.ClusterConfig, patch string) (*k0sconfig.C } // InstallFlags returns a list of default flags to be used when bootstrapping a k0s cluster. -func InstallFlags(nodeIP string, cfg *k0sconfig.ClusterConfig) ([]string, error) { +func InstallFlags(nodeIP string, cfg *k0sconfig.ClusterConfig) []string { flags := []string{ "install", "controller", @@ -106,7 +106,7 @@ func InstallFlags(nodeIP string, cfg *k0sconfig.ClusterConfig) ([]string, error) flags = append(flags, ProfileInstallFlag(cfg)) flags = append(flags, AdditionalInstallFlags(nodeIP)...) flags = append(flags, AdditionalInstallFlagsController()...) - return flags, nil + return flags } func AdditionalInstallFlags(nodeIP string) []string { diff --git a/pkg/k0s/install.go b/pkg/k0s/install.go index 3c70bc271..3748f1c9f 100644 --- a/pkg/k0s/install.go +++ b/pkg/k0s/install.go @@ -31,7 +31,7 @@ func Install(networkInterface string, cfg *k0sv1beta1.ClusterConfig) error { if err != nil { return fmt.Errorf("unable to find first valid address: %w", err) } - flags, err := config.InstallFlags(nodeIP, cfg) + flags := config.InstallFlags(nodeIP, cfg) if err != nil { return fmt.Errorf("unable to get install flags: %w", err) } diff --git a/pkg/kotsadm/types.go b/pkg/kotsadm/types.go index b5c87244f..d93de3ec5 100644 --- a/pkg/kotsadm/types.go +++ b/pkg/kotsadm/types.go @@ -35,6 +35,9 @@ func (j JoinCommandResponse) extractK0sConfigOverridePatch(data []byte) (dig.Map if storage := config.DigMapping("config", "spec", "storage"); len(storage) > 0 { result.DigMapping("config", "spec")["storage"] = storage } + if workerProfiles := config.DigMapping("config", "spec", "workerProfiles"); len(workerProfiles) > 0 { + result.DigMapping("config", "spec")["workerProfiles"] = workerProfiles + } return result, nil } From 33512a024f477ea6fc62562ba2a19f3994f936dc Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Thu, 20 Mar 2025 16:35:26 +0000 Subject: [PATCH 21/48] generate --- kinds/apis/v1beta1/zz_generated.deepcopy.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/kinds/apis/v1beta1/zz_generated.deepcopy.go b/kinds/apis/v1beta1/zz_generated.deepcopy.go index 066b5040d..28c1467c8 100644 --- a/kinds/apis/v1beta1/zz_generated.deepcopy.go +++ b/kinds/apis/v1beta1/zz_generated.deepcopy.go @@ -637,13 +637,6 @@ func (in *UnsupportedOverrides) DeepCopyInto(out *UnsupportedOverrides) { *out = make([]BuiltInExtension, len(*in)) copy(*out, *in) } - if in.WorkerProfiles != nil { - in, out := &in.WorkerProfiles, &out.WorkerProfiles - *out = make([]k0sv1beta1.WorkerProfile, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UnsupportedOverrides. From 8bbb8946d0276d212b05e308edd7b4f3f5d37cfd Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Thu, 20 Mar 2025 16:35:49 +0000 Subject: [PATCH 22/48] manifests --- .../charts/crds/templates/resources.yaml | 32 ------------------- ...mbeddedcluster.replicated.com_configs.yaml | 17 ---------- ...dcluster.replicated.com_installations.yaml | 17 ---------- 3 files changed, 66 deletions(-) diff --git a/operator/charts/embedded-cluster-operator/charts/crds/templates/resources.yaml b/operator/charts/embedded-cluster-operator/charts/crds/templates/resources.yaml index b10cb974b..72dbb4d59 100644 --- a/operator/charts/embedded-cluster-operator/charts/crds/templates/resources.yaml +++ b/operator/charts/embedded-cluster-operator/charts/crds/templates/resources.yaml @@ -226,22 +226,6 @@ spec: layout inside this configuration is very dynamic we have chosen to use a string here. type: string - workerProfiles: - description: |- - WorkerProfiles holds the worker profiles used to configure the cluster. - The profile named "default" will be applied by default. - items: - description: WorkerProfile worker profile - properties: - name: - description: String; name to use as profile selector for the worker process - type: string - values: - description: Worker Mapping object - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - type: array type: object version: type: string @@ -541,22 +525,6 @@ spec: layout inside this configuration is very dynamic we have chosen to use a string here. type: string - workerProfiles: - description: |- - WorkerProfiles holds the worker profiles used to configure the cluster. - The profile named "default" will be applied by default. - items: - description: WorkerProfile worker profile - properties: - name: - description: String; name to use as profile selector for the worker process - type: string - values: - description: Worker Mapping object - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - type: array type: object version: type: string diff --git a/operator/config/crd/bases/embeddedcluster.replicated.com_configs.yaml b/operator/config/crd/bases/embeddedcluster.replicated.com_configs.yaml index ccbf1735a..c3bb59e32 100644 --- a/operator/config/crd/bases/embeddedcluster.replicated.com_configs.yaml +++ b/operator/config/crd/bases/embeddedcluster.replicated.com_configs.yaml @@ -235,23 +235,6 @@ spec: layout inside this configuration is very dynamic we have chosen to use a string here. type: string - workerProfiles: - description: |- - WorkerProfiles holds the worker profiles used to configure the cluster. - The profile named "default" will be applied by default. - items: - description: WorkerProfile worker profile - properties: - name: - description: String; name to use as profile selector for - the worker process - type: string - values: - description: Worker Mapping object - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - type: array type: object version: type: string diff --git a/operator/config/crd/bases/embeddedcluster.replicated.com_installations.yaml b/operator/config/crd/bases/embeddedcluster.replicated.com_installations.yaml index e4c429f64..0288417c6 100644 --- a/operator/config/crd/bases/embeddedcluster.replicated.com_installations.yaml +++ b/operator/config/crd/bases/embeddedcluster.replicated.com_installations.yaml @@ -297,23 +297,6 @@ spec: layout inside this configuration is very dynamic we have chosen to use a string here. type: string - workerProfiles: - description: |- - WorkerProfiles holds the worker profiles used to configure the cluster. - The profile named "default" will be applied by default. - items: - description: WorkerProfile worker profile - properties: - name: - description: String; name to use as profile selector - for the worker process - type: string - values: - description: Worker Mapping object - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - type: array type: object version: type: string From 3ddbe2b30faf8f005a5ff14559c44c6ea0a79ba3 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Thu, 20 Mar 2025 16:38:23 +0000 Subject: [PATCH 23/48] load profiles from config on disk --- cmd/installer/cli/join.go | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index 32a249bce..7236dd054 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -451,29 +451,27 @@ func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error { } // getFirstDefinedProfileFlag returns the name of the first defined worker profile -// from the returned join command config +// from the k0s config file on disk func getFirstDefinedProfileFlag(jcmd *kotsadm.JoinCommandResponse) string { - if jcmd == nil || jcmd.InstallationSpec.Config == nil { - return "" - } - if jcmd.InstallationSpec.Config.UnsupportedOverrides.K0s == "" { - return "" - } - - cfgString, err := config.ExtractK0sConfigPatch(jcmd.InstallationSpec.Config.UnsupportedOverrides.K0s) + data, err := os.ReadFile(runtimeconfig.PathToK0sConfig()) if err != nil { - logrus.Debugf("unable to extract k0s config from installation object: %v", err) + logrus.Debugf("getFirstDefinedProfileFlag: unable to read k0s config file: %v", err) return "" } var k0scfg k0sv1beta1.ClusterConfig - if err := yaml.Unmarshal([]byte(cfgString), &k0scfg); err != nil { - logrus.Debugf("unable to parse k0s config: %v", err) + if err := yaml.Unmarshal(data, &k0scfg); err != nil { + logrus.Debugf("getFirstDefinedProfileFlag: unable to parse k0s config: %v", err) + return "" + } + if k0scfg.Spec == nil { + logrus.Debugf("getFirstDefinedProfileFlag: k0s config Spec is nil") return "" } - fmt.Printf("%+v\n", k0scfg.Spec) - if k0scfg.Spec == nil || len(k0scfg.Spec.WorkerProfiles) == 0 { + if len(k0scfg.Spec.WorkerProfiles) == 0 { + logrus.Debugf("getFirstDefinedProfileFlag: no worker profiles found") return "" } + logrus.Debugf("getFirstDefinedProfileFlag: found worker profile: %s", k0scfg.Spec.WorkerProfiles[0].Name) return k0scfg.Spec.WorkerProfiles[0].Name } From f53b8999f3fb8a16e2cc4e511fd3c18be2782517 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Thu, 20 Mar 2025 16:41:16 +0000 Subject: [PATCH 24/48] manifests --- .../config-embeddedcluster-v1beta1.json | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/operator/schemas/config-embeddedcluster-v1beta1.json b/operator/schemas/config-embeddedcluster-v1beta1.json index 48f8bb61f..20de5bd3a 100644 --- a/operator/schemas/config-embeddedcluster-v1beta1.json +++ b/operator/schemas/config-embeddedcluster-v1beta1.json @@ -256,25 +256,6 @@ "k0s": { "description": "K0s holds the overrides used to configure k0s. These overrides\nare merged on top of the default k0s configuration. As the data\nlayout inside this configuration is very dynamic we have chosen\nto use a string here.", "type": "string" - }, - "workerProfiles": { - "description": "WorkerProfiles holds the worker profiles used to configure the cluster.\nThe profile named \"default\" will be applied by default.", - "type": "array", - "items": { - "description": "WorkerProfile worker profile", - "type": "object", - "properties": { - "name": { - "description": "String; name to use as profile selector for the worker process", - "type": "string" - }, - "values": { - "description": "Worker Mapping object", - "type": "object", - "x-kubernetes-preserve-unknown-fields": true - } - } - } } } }, From 76f99524840e6313f07e63731be9d1b43d387123 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Thu, 20 Mar 2025 16:46:14 +0000 Subject: [PATCH 25/48] remove unused err check --- pkg/k0s/install.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/k0s/install.go b/pkg/k0s/install.go index 3748f1c9f..76c0bbdee 100644 --- a/pkg/k0s/install.go +++ b/pkg/k0s/install.go @@ -32,9 +32,6 @@ func Install(networkInterface string, cfg *k0sv1beta1.ClusterConfig) error { return fmt.Errorf("unable to find first valid address: %w", err) } flags := config.InstallFlags(nodeIP, cfg) - if err != nil { - return fmt.Errorf("unable to get install flags: %w", err) - } if _, err := helpers.RunCommand(hstbin, flags...); err != nil { return fmt.Errorf("unable to install: %w", err) } From 87a7c83bb3904598efc16aad40eeda5ae0c0c0d5 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Thu, 20 Mar 2025 17:22:52 +0000 Subject: [PATCH 26/48] vibe coding --- cmd/installer/cli/join.go | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index 7236dd054..577a09156 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -422,6 +422,7 @@ func startAndWaitForK0s(ctx context.Context, name string, jcmd *kotsadm.JoinComm // applyJoinConfigurationOverrides applies both config overrides received from the kots api. // Applies first the EmbeddedOverrides and then the EndUserOverrides. func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error { + fmt.Println(jcmd.InstallationSpec.Config.UnsupportedOverrides.K0s) patch, err := jcmd.EmbeddedOverrides() if err != nil { return fmt.Errorf("unable to get embedded overrides: %w", err) @@ -451,26 +452,47 @@ func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error { } // getFirstDefinedProfileFlag returns the name of the first defined worker profile -// from the k0s config file on disk +// from the join command response func getFirstDefinedProfileFlag(jcmd *kotsadm.JoinCommandResponse) string { - data, err := os.ReadFile(runtimeconfig.PathToK0sConfig()) + if jcmd == nil { + logrus.Debugf("getFirstDefinedProfileFlag: no join command response") + return "" + } + + patch, err := jcmd.EmbeddedOverrides() if err != nil { - logrus.Debugf("getFirstDefinedProfileFlag: unable to read k0s config file: %v", err) + logrus.Debugf("getFirstDefinedProfileFlag: unable to get embedded overrides: %v", err) return "" } + + if len(patch) == 0 { + logrus.Debugf("getFirstDefinedProfileFlag: no overrides found") + return "" + } + + // Extract the k0s config from under the "config" key + configData, err := yaml.Marshal(patch["config"]) + if err != nil { + logrus.Debugf("getFirstDefinedProfileFlag: unable to marshal config: %v", err) + return "" + } + var k0scfg k0sv1beta1.ClusterConfig - if err := yaml.Unmarshal(data, &k0scfg); err != nil { + if err := yaml.Unmarshal(configData, &k0scfg); err != nil { logrus.Debugf("getFirstDefinedProfileFlag: unable to parse k0s config: %v", err) return "" } + if k0scfg.Spec == nil { logrus.Debugf("getFirstDefinedProfileFlag: k0s config Spec is nil") return "" } + if len(k0scfg.Spec.WorkerProfiles) == 0 { logrus.Debugf("getFirstDefinedProfileFlag: no worker profiles found") return "" } + logrus.Debugf("getFirstDefinedProfileFlag: found worker profile: %s", k0scfg.Spec.WorkerProfiles[0].Name) return k0scfg.Spec.WorkerProfiles[0].Name } From 8cb65b4a37be8716403f3e4ec30c6b52c6fdbb2c Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Fri, 21 Mar 2025 09:12:10 +0000 Subject: [PATCH 27/48] also include worker profiles when patching config --- cmd/installer/cli/join.go | 1 - pkg/k0s/install.go | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index 577a09156..3e3de93d7 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -422,7 +422,6 @@ func startAndWaitForK0s(ctx context.Context, name string, jcmd *kotsadm.JoinComm // applyJoinConfigurationOverrides applies both config overrides received from the kots api. // Applies first the EmbeddedOverrides and then the EndUserOverrides. func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error { - fmt.Println(jcmd.InstallationSpec.Config.UnsupportedOverrides.K0s) patch, err := jcmd.EmbeddedOverrides() if err != nil { return fmt.Errorf("unable to get embedded overrides: %w", err) diff --git a/pkg/k0s/install.go b/pkg/k0s/install.go index 76c0bbdee..f79eb5f60 100644 --- a/pkg/k0s/install.go +++ b/pkg/k0s/install.go @@ -182,6 +182,12 @@ func PatchK0sConfig(path string, patch string) error { } finalcfg.Spec.Storage = result.Spec.Storage } + if result.Spec.WorkerProfiles != nil { + if finalcfg.Spec == nil { + finalcfg.Spec = &k0sv1beta1.ClusterSpec{} + } + finalcfg.Spec.WorkerProfiles = result.Spec.WorkerProfiles + } // This is necessary to install the previous version of k0s in e2e tests // TODO: remove this once the previous version is > 1.29 unstructured, err := helpers.K0sClusterConfigTo129Compat(&finalcfg) From 3a1fac1b88f816a07aff991a8495522cfee2233d Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Fri, 21 Mar 2025 09:50:58 +0000 Subject: [PATCH 28/48] debug loggin --- cmd/installer/cli/join.go | 2 ++ pkg/k0s/install.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index 3e3de93d7..52b21ffd6 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -423,6 +423,8 @@ func startAndWaitForK0s(ctx context.Context, name string, jcmd *kotsadm.JoinComm // Applies first the EmbeddedOverrides and then the EndUserOverrides. func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error { patch, err := jcmd.EmbeddedOverrides() + fmt.Println(patch) + fmt.Println("--------------------------------") if err != nil { return fmt.Errorf("unable to get embedded overrides: %w", err) } diff --git a/pkg/k0s/install.go b/pkg/k0s/install.go index f79eb5f60..203929219 100644 --- a/pkg/k0s/install.go +++ b/pkg/k0s/install.go @@ -195,6 +195,8 @@ func PatchK0sConfig(path string, patch string) error { return fmt.Errorf("unable to convert cluster config to 1.29 compat: %w", err) } data, err := k8syaml.Marshal(unstructured) + fmt.Println(string(data)) + fmt.Println("--------------------------------") if err != nil { return fmt.Errorf("unable to marshal node config: %w", err) } From d290c1e0143a290a6b76390b1da92a2a8dea3144 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Fri, 21 Mar 2025 10:17:32 +0000 Subject: [PATCH 29/48] debug loggin --- pkg/kotsadm/types.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/kotsadm/types.go b/pkg/kotsadm/types.go index d93de3ec5..af22c8b0f 100644 --- a/pkg/kotsadm/types.go +++ b/pkg/kotsadm/types.go @@ -28,6 +28,8 @@ func (j JoinCommandResponse) extractK0sConfigOverridePatch(data []byte) (dig.Map if err := yaml.Unmarshal(data, &config); err != nil { return nil, fmt.Errorf("unable to unmarshal embedded config: %w", err) } + fmt.Println(config) + fmt.Println("--------------------------------") result := dig.Mapping{} if api := config.DigMapping("config", "spec", "api"); len(api) > 0 { result.DigMapping("config", "spec")["api"] = api @@ -38,6 +40,8 @@ func (j JoinCommandResponse) extractK0sConfigOverridePatch(data []byte) (dig.Map if workerProfiles := config.DigMapping("config", "spec", "workerProfiles"); len(workerProfiles) > 0 { result.DigMapping("config", "spec")["workerProfiles"] = workerProfiles } + fmt.Println(result) + fmt.Println("--------------------------------") return result, nil } From 490a601a6d801bd3f84bcf5beb10eabbea19b126 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Fri, 21 Mar 2025 10:47:39 +0000 Subject: [PATCH 30/48] more debug logging --- cmd/installer/cli/join.go | 31 ++++++++----------------------- pkg/k0s/install.go | 4 ++++ pkg/kotsadm/types.go | 2 ++ 3 files changed, 14 insertions(+), 23 deletions(-) diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index 52b21ffd6..951724346 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -332,7 +332,7 @@ func installAndJoinCluster(ctx context.Context, jcmd *kotsadm.JoinCommandRespons } logrus.Debugf("joining node to cluster") - if err := runK0sInstallCommand(flags.networkInterface, jcmd.K0sJoinCommand, getFirstDefinedProfileFlag(jcmd)); err != nil { + if err := runK0sInstallCommand(flags.networkInterface, jcmd.K0sJoinCommand, getFirstDefinedProfileFlag()); err != nil { return fmt.Errorf("unable to join node to cluster: %w", err) } @@ -422,7 +422,9 @@ func startAndWaitForK0s(ctx context.Context, name string, jcmd *kotsadm.JoinComm // applyJoinConfigurationOverrides applies both config overrides received from the kots api. // Applies first the EmbeddedOverrides and then the EndUserOverrides. func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error { + fmt.Println("applyJoinConfigurationOverrides") patch, err := jcmd.EmbeddedOverrides() + fmt.Println("patch from embedded overrides") fmt.Println(patch) fmt.Println("--------------------------------") if err != nil { @@ -453,33 +455,16 @@ func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error { } // getFirstDefinedProfileFlag returns the name of the first defined worker profile -// from the join command response -func getFirstDefinedProfileFlag(jcmd *kotsadm.JoinCommandResponse) string { - if jcmd == nil { - logrus.Debugf("getFirstDefinedProfileFlag: no join command response") - return "" - } - - patch, err := jcmd.EmbeddedOverrides() - if err != nil { - logrus.Debugf("getFirstDefinedProfileFlag: unable to get embedded overrides: %v", err) - return "" - } - - if len(patch) == 0 { - logrus.Debugf("getFirstDefinedProfileFlag: no overrides found") - return "" - } - - // Extract the k0s config from under the "config" key - configData, err := yaml.Marshal(patch["config"]) +// from the on-disk k0s config file +func getFirstDefinedProfileFlag() string { + data, err := os.ReadFile(runtimeconfig.PathToK0sConfig()) if err != nil { - logrus.Debugf("getFirstDefinedProfileFlag: unable to marshal config: %v", err) + logrus.Debugf("getFirstDefinedProfileFlag: unable to read k0s config file: %v", err) return "" } var k0scfg k0sv1beta1.ClusterConfig - if err := yaml.Unmarshal(configData, &k0scfg); err != nil { + if err := yaml.Unmarshal(data, &k0scfg); err != nil { logrus.Debugf("getFirstDefinedProfileFlag: unable to parse k0s config: %v", err) return "" } diff --git a/pkg/k0s/install.go b/pkg/k0s/install.go index 203929219..882bc7fdc 100644 --- a/pkg/k0s/install.go +++ b/pkg/k0s/install.go @@ -150,6 +150,9 @@ func applyUnsupportedOverrides(cfg *k0sv1beta1.ClusterConfig, overrides string) // PatchK0sConfig patches the created k0s config with the unsupported overrides passed in. func PatchK0sConfig(path string, patch string) error { + fmt.Println("PatchK0sConfig") + fmt.Println(patch) + fmt.Println("--------------------------------") if len(patch) == 0 { return nil } @@ -195,6 +198,7 @@ func PatchK0sConfig(path string, patch string) error { return fmt.Errorf("unable to convert cluster config to 1.29 compat: %w", err) } data, err := k8syaml.Marshal(unstructured) + fmt.Println("config data after patch") fmt.Println(string(data)) fmt.Println("--------------------------------") if err != nil { diff --git a/pkg/kotsadm/types.go b/pkg/kotsadm/types.go index af22c8b0f..a5df46068 100644 --- a/pkg/kotsadm/types.go +++ b/pkg/kotsadm/types.go @@ -28,6 +28,7 @@ func (j JoinCommandResponse) extractK0sConfigOverridePatch(data []byte) (dig.Map if err := yaml.Unmarshal(data, &config); err != nil { return nil, fmt.Errorf("unable to unmarshal embedded config: %w", err) } + fmt.Println("config") fmt.Println(config) fmt.Println("--------------------------------") result := dig.Mapping{} @@ -40,6 +41,7 @@ func (j JoinCommandResponse) extractK0sConfigOverridePatch(data []byte) (dig.Map if workerProfiles := config.DigMapping("config", "spec", "workerProfiles"); len(workerProfiles) > 0 { result.DigMapping("config", "spec")["workerProfiles"] = workerProfiles } + fmt.Println("result") fmt.Println(result) fmt.Println("--------------------------------") return result, nil From 6209ad663d27a6c16973218dfa607b2489599b27 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Fri, 21 Mar 2025 11:18:08 +0000 Subject: [PATCH 31/48] more debug logging --- pkg/kotsadm/types.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/kotsadm/types.go b/pkg/kotsadm/types.go index a5df46068..973fbdeda 100644 --- a/pkg/kotsadm/types.go +++ b/pkg/kotsadm/types.go @@ -38,7 +38,11 @@ func (j JoinCommandResponse) extractK0sConfigOverridePatch(data []byte) (dig.Map if storage := config.DigMapping("config", "spec", "storage"); len(storage) > 0 { result.DigMapping("config", "spec")["storage"] = storage } - if workerProfiles := config.DigMapping("config", "spec", "workerProfiles"); len(workerProfiles) > 0 { + workerProfiles := config.DigMapping("config", "spec", "workerProfiles") + fmt.Println("workerProfiles") + fmt.Println(workerProfiles) + fmt.Println("--------------------------------") + if len(workerProfiles) > 0 { result.DigMapping("config", "spec")["workerProfiles"] = workerProfiles } fmt.Println("result") From 024304eae34a509c09e8370fe954864435ff4088 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Fri, 21 Mar 2025 11:51:03 +0000 Subject: [PATCH 32/48] try Dig over DigMapping --- pkg/kotsadm/types.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/kotsadm/types.go b/pkg/kotsadm/types.go index 973fbdeda..5750f518d 100644 --- a/pkg/kotsadm/types.go +++ b/pkg/kotsadm/types.go @@ -38,11 +38,11 @@ func (j JoinCommandResponse) extractK0sConfigOverridePatch(data []byte) (dig.Map if storage := config.DigMapping("config", "spec", "storage"); len(storage) > 0 { result.DigMapping("config", "spec")["storage"] = storage } - workerProfiles := config.DigMapping("config", "spec", "workerProfiles") + workerProfiles := config.Dig("config", "spec", "workerProfiles") fmt.Println("workerProfiles") fmt.Println(workerProfiles) fmt.Println("--------------------------------") - if len(workerProfiles) > 0 { + if workerProfiles != nil { result.DigMapping("config", "spec")["workerProfiles"] = workerProfiles } fmt.Println("result") From 11a3b2cc0a53acde017da021f0a18ea4c1e320f3 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Fri, 21 Mar 2025 12:21:22 +0000 Subject: [PATCH 33/48] log patched config --- cmd/installer/cli/join.go | 14 +++++++++----- pkg/k0s/install.go | 6 ------ pkg/kotsadm/types.go | 9 --------- 3 files changed, 9 insertions(+), 20 deletions(-) diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index 951724346..b95c47146 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -331,8 +331,13 @@ func installAndJoinCluster(ctx context.Context, jcmd *kotsadm.JoinCommandRespons return fmt.Errorf("unable to apply configuration overrides: %w", err) } + profile := getFirstDefinedProfileFlag() + fmt.Println("profile") + fmt.Println(profile) + fmt.Println("--------------------------------") + logrus.Debugf("joining node to cluster") - if err := runK0sInstallCommand(flags.networkInterface, jcmd.K0sJoinCommand, getFirstDefinedProfileFlag()); err != nil { + if err := runK0sInstallCommand(flags.networkInterface, jcmd.K0sJoinCommand, profile); err != nil { return fmt.Errorf("unable to join node to cluster: %w", err) } @@ -422,11 +427,7 @@ func startAndWaitForK0s(ctx context.Context, name string, jcmd *kotsadm.JoinComm // applyJoinConfigurationOverrides applies both config overrides received from the kots api. // Applies first the EmbeddedOverrides and then the EndUserOverrides. func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error { - fmt.Println("applyJoinConfigurationOverrides") patch, err := jcmd.EmbeddedOverrides() - fmt.Println("patch from embedded overrides") - fmt.Println(patch) - fmt.Println("--------------------------------") if err != nil { return fmt.Errorf("unable to get embedded overrides: %w", err) } @@ -468,6 +469,9 @@ func getFirstDefinedProfileFlag() string { logrus.Debugf("getFirstDefinedProfileFlag: unable to parse k0s config: %v", err) return "" } + fmt.Println("k0scfg") + fmt.Printf("%+v\n", k0scfg) + fmt.Println("--------------------------------") if k0scfg.Spec == nil { logrus.Debugf("getFirstDefinedProfileFlag: k0s config Spec is nil") diff --git a/pkg/k0s/install.go b/pkg/k0s/install.go index 882bc7fdc..f79eb5f60 100644 --- a/pkg/k0s/install.go +++ b/pkg/k0s/install.go @@ -150,9 +150,6 @@ func applyUnsupportedOverrides(cfg *k0sv1beta1.ClusterConfig, overrides string) // PatchK0sConfig patches the created k0s config with the unsupported overrides passed in. func PatchK0sConfig(path string, patch string) error { - fmt.Println("PatchK0sConfig") - fmt.Println(patch) - fmt.Println("--------------------------------") if len(patch) == 0 { return nil } @@ -198,9 +195,6 @@ func PatchK0sConfig(path string, patch string) error { return fmt.Errorf("unable to convert cluster config to 1.29 compat: %w", err) } data, err := k8syaml.Marshal(unstructured) - fmt.Println("config data after patch") - fmt.Println(string(data)) - fmt.Println("--------------------------------") if err != nil { return fmt.Errorf("unable to marshal node config: %w", err) } diff --git a/pkg/kotsadm/types.go b/pkg/kotsadm/types.go index 5750f518d..e7ef813e3 100644 --- a/pkg/kotsadm/types.go +++ b/pkg/kotsadm/types.go @@ -28,9 +28,6 @@ func (j JoinCommandResponse) extractK0sConfigOverridePatch(data []byte) (dig.Map if err := yaml.Unmarshal(data, &config); err != nil { return nil, fmt.Errorf("unable to unmarshal embedded config: %w", err) } - fmt.Println("config") - fmt.Println(config) - fmt.Println("--------------------------------") result := dig.Mapping{} if api := config.DigMapping("config", "spec", "api"); len(api) > 0 { result.DigMapping("config", "spec")["api"] = api @@ -39,15 +36,9 @@ func (j JoinCommandResponse) extractK0sConfigOverridePatch(data []byte) (dig.Map result.DigMapping("config", "spec")["storage"] = storage } workerProfiles := config.Dig("config", "spec", "workerProfiles") - fmt.Println("workerProfiles") - fmt.Println(workerProfiles) - fmt.Println("--------------------------------") if workerProfiles != nil { result.DigMapping("config", "spec")["workerProfiles"] = workerProfiles } - fmt.Println("result") - fmt.Println(result) - fmt.Println("--------------------------------") return result, nil } From 980ac37bc0b2699400aec95b09633f2767e4120d Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Fri, 21 Mar 2025 13:04:37 +0000 Subject: [PATCH 34/48] make sure config is synced to disk before reading back to get profiles --- cmd/installer/cli/join.go | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index b95c47146..dab5ce69e 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "os" "strings" "syscall" @@ -331,13 +332,8 @@ func installAndJoinCluster(ctx context.Context, jcmd *kotsadm.JoinCommandRespons return fmt.Errorf("unable to apply configuration overrides: %w", err) } - profile := getFirstDefinedProfileFlag() - fmt.Println("profile") - fmt.Println(profile) - fmt.Println("--------------------------------") - logrus.Debugf("joining node to cluster") - if err := runK0sInstallCommand(flags.networkInterface, jcmd.K0sJoinCommand, profile); err != nil { + if err := runK0sInstallCommand(flags.networkInterface, jcmd.K0sJoinCommand); err != nil { return fmt.Errorf("unable to join node to cluster: %w", err) } @@ -455,10 +451,26 @@ func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error { return nil } +// readK0sConfig ensures the file has been synced to disk before reading it +func readK0sConfig() ([]byte, error) { + file, err := os.OpenFile(runtimeconfig.PathToK0sConfig(), os.O_RDONLY, 0644) + if err != nil { + return nil, fmt.Errorf("unable to open k0s config file: %w", err) + } + defer file.Close() + + // Ensure file is synced to disk + if err := file.Sync(); err != nil { + return nil, fmt.Errorf("unable to sync k0s config file: %w", err) + } + + return io.ReadAll(file) +} + // getFirstDefinedProfileFlag returns the name of the first defined worker profile // from the on-disk k0s config file func getFirstDefinedProfileFlag() string { - data, err := os.ReadFile(runtimeconfig.PathToK0sConfig()) + data, err := readK0sConfig() if err != nil { logrus.Debugf("getFirstDefinedProfileFlag: unable to read k0s config file: %v", err) return "" @@ -470,7 +482,7 @@ func getFirstDefinedProfileFlag() string { return "" } fmt.Println("k0scfg") - fmt.Printf("%+v\n", k0scfg) + fmt.Printf("%+v\n", k0scfg.Spec.WorkerProfiles) fmt.Println("--------------------------------") if k0scfg.Spec == nil { @@ -489,10 +501,11 @@ func getFirstDefinedProfileFlag() string { // runK0sInstallCommand runs the k0s install command as provided by the kots // adm api. -func runK0sInstallCommand(networkInterface string, fullcmd string, profile string) error { +func runK0sInstallCommand(networkInterface string, fullcmd string) error { args := strings.Split(fullcmd, " ") args = append(args, "--token-file", "/etc/k0s/join-token") + profile := getFirstDefinedProfileFlag() if profile != "" { args = append(args, "--profile", profile) } From 89f65c8d52eb29611d7296f198be6297fc8c0269 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Fri, 21 Mar 2025 13:43:24 +0000 Subject: [PATCH 35/48] verbose logging of config loading? --- cmd/installer/cli/join.go | 57 +++++++++++---------------------------- 1 file changed, 15 insertions(+), 42 deletions(-) diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index dab5ce69e..4910bf81c 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "io" "os" "strings" "syscall" @@ -451,51 +450,25 @@ func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error { return nil } -// readK0sConfig ensures the file has been synced to disk before reading it -func readK0sConfig() ([]byte, error) { - file, err := os.OpenFile(runtimeconfig.PathToK0sConfig(), os.O_RDONLY, 0644) - if err != nil { - return nil, fmt.Errorf("unable to open k0s config file: %w", err) - } - defer file.Close() - - // Ensure file is synced to disk - if err := file.Sync(); err != nil { - return nil, fmt.Errorf("unable to sync k0s config file: %w", err) - } - - return io.ReadAll(file) -} - -// getFirstDefinedProfileFlag returns the name of the first defined worker profile +// getFirstDefinedProfile returns the name of the first defined worker profile // from the on-disk k0s config file -func getFirstDefinedProfileFlag() string { - data, err := readK0sConfig() +func getFirstDefinedProfile() string { + cfg, err := os.ReadFile(runtimeconfig.PathToK0sConfig()) + fmt.Printf("cfg: %+v\n", cfg) if err != nil { - logrus.Debugf("getFirstDefinedProfileFlag: unable to read k0s config file: %v", err) return "" } - - var k0scfg k0sv1beta1.ClusterConfig - if err := yaml.Unmarshal(data, &k0scfg); err != nil { - logrus.Debugf("getFirstDefinedProfileFlag: unable to parse k0s config: %v", err) - return "" - } - fmt.Println("k0scfg") - fmt.Printf("%+v\n", k0scfg.Spec.WorkerProfiles) - fmt.Println("--------------------------------") - - if k0scfg.Spec == nil { - logrus.Debugf("getFirstDefinedProfileFlag: k0s config Spec is nil") + k0scfg := k0sv1beta1.ClusterConfig{} + if err := yaml.Unmarshal(cfg, &k0scfg); err != nil { return "" } - + fmt.Printf("k0scfg: %+v\n", k0scfg) + fmt.Printf("k0scfg.Spec: %+v\n", k0scfg.Spec) + fmt.Printf("k0scfg.Spec.WorkerProfiles: %+v\n", k0scfg.Spec.WorkerProfiles) + // get first worker profile from cfg if len(k0scfg.Spec.WorkerProfiles) == 0 { - logrus.Debugf("getFirstDefinedProfileFlag: no worker profiles found") return "" } - - logrus.Debugf("getFirstDefinedProfileFlag: found worker profile: %s", k0scfg.Spec.WorkerProfiles[0].Name) return k0scfg.Spec.WorkerProfiles[0].Name } @@ -505,16 +478,16 @@ func runK0sInstallCommand(networkInterface string, fullcmd string) error { args := strings.Split(fullcmd, " ") args = append(args, "--token-file", "/etc/k0s/join-token") - profile := getFirstDefinedProfileFlag() - if profile != "" { - args = append(args, "--profile", profile) - } - nodeIP, err := netutils.FirstValidAddress(networkInterface) if err != nil { return fmt.Errorf("unable to find first valid address: %w", err) } + profile := getFirstDefinedProfile() + if profile != "" { + args = append(args, "--profile", profile) + } + args = append(args, config.AdditionalInstallFlags(nodeIP)...) if strings.Contains(fullcmd, "controller") { From 71fee40e9d25ba8eb25c4cfc525fff728208ba45 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Fri, 21 Mar 2025 14:33:18 +0000 Subject: [PATCH 36/48] try k0s own config reader --- cmd/installer/cli/join.go | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index 4910bf81c..203b023a0 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -8,7 +8,7 @@ import ( "strings" "syscall" - k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" + k0sconfig "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" "github.com/replicatedhq/embedded-cluster/pkg/addons" "github.com/replicatedhq/embedded-cluster/pkg/airgap" @@ -331,8 +331,10 @@ func installAndJoinCluster(ctx context.Context, jcmd *kotsadm.JoinCommandRespons return fmt.Errorf("unable to apply configuration overrides: %w", err) } + profile := getFirstDefinedProfile() + logrus.Debugf("joining node to cluster") - if err := runK0sInstallCommand(flags.networkInterface, jcmd.K0sJoinCommand); err != nil { + if err := runK0sInstallCommand(flags.networkInterface, jcmd.K0sJoinCommand, profile); err != nil { return fmt.Errorf("unable to join node to cluster: %w", err) } @@ -450,31 +452,27 @@ func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error { return nil } -// getFirstDefinedProfile returns the name of the first defined worker profile -// from the on-disk k0s config file func getFirstDefinedProfile() string { - cfg, err := os.ReadFile(runtimeconfig.PathToK0sConfig()) - fmt.Printf("cfg: %+v\n", cfg) + k0scfg, err := os.Open(runtimeconfig.PathToK0sConfig()) if err != nil { return "" } - k0scfg := k0sv1beta1.ClusterConfig{} - if err := yaml.Unmarshal(cfg, &k0scfg); err != nil { + defer k0scfg.Close() + cfg, err := k0sconfig.ConfigFromReader(k0scfg) + fmt.Printf("cfg: %+v\n", cfg) + fmt.Printf("cfg.Spec: %+v\n", cfg.Spec) + fmt.Printf("cfg.Spec.WorkerProfiles: %+v\n", cfg.Spec.WorkerProfiles) + if err != nil { return "" } - fmt.Printf("k0scfg: %+v\n", k0scfg) - fmt.Printf("k0scfg.Spec: %+v\n", k0scfg.Spec) - fmt.Printf("k0scfg.Spec.WorkerProfiles: %+v\n", k0scfg.Spec.WorkerProfiles) - // get first worker profile from cfg - if len(k0scfg.Spec.WorkerProfiles) == 0 { - return "" + if len(cfg.Spec.WorkerProfiles) > 0 { + return cfg.Spec.WorkerProfiles[0].Name } - return k0scfg.Spec.WorkerProfiles[0].Name + return "" } // runK0sInstallCommand runs the k0s install command as provided by the kots -// adm api. -func runK0sInstallCommand(networkInterface string, fullcmd string) error { +func runK0sInstallCommand(networkInterface string, fullcmd string, profile string) error { args := strings.Split(fullcmd, " ") args = append(args, "--token-file", "/etc/k0s/join-token") @@ -483,7 +481,6 @@ func runK0sInstallCommand(networkInterface string, fullcmd string) error { return fmt.Errorf("unable to find first valid address: %w", err) } - profile := getFirstDefinedProfile() if profile != "" { args = append(args, "--profile", profile) } From a2662dce8e1a43330742dd75eca59a6d449ac46e Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Fri, 21 Mar 2025 14:56:07 +0000 Subject: [PATCH 37/48] proper debug logging --- cmd/installer/cli/join.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index 203b023a0..f884e5932 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -455,14 +455,13 @@ func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error { func getFirstDefinedProfile() string { k0scfg, err := os.Open(runtimeconfig.PathToK0sConfig()) if err != nil { + logrus.Errorf("unable to open k0s config: %v", err) return "" } defer k0scfg.Close() cfg, err := k0sconfig.ConfigFromReader(k0scfg) - fmt.Printf("cfg: %+v\n", cfg) - fmt.Printf("cfg.Spec: %+v\n", cfg.Spec) - fmt.Printf("cfg.Spec.WorkerProfiles: %+v\n", cfg.Spec.WorkerProfiles) if err != nil { + logrus.Errorf("unable to parse k0s config: %v", err) return "" } if len(cfg.Spec.WorkerProfiles) > 0 { From 4c39791194759fc008157b08d798598aae494df4 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Fri, 21 Mar 2025 14:58:47 +0000 Subject: [PATCH 38/48] debug not error --- cmd/installer/cli/join.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index f884e5932..822ff76f4 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -455,13 +455,13 @@ func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error { func getFirstDefinedProfile() string { k0scfg, err := os.Open(runtimeconfig.PathToK0sConfig()) if err != nil { - logrus.Errorf("unable to open k0s config: %v", err) + logrus.Debugf("unable to open k0s config: %v", err) return "" } defer k0scfg.Close() cfg, err := k0sconfig.ConfigFromReader(k0scfg) if err != nil { - logrus.Errorf("unable to parse k0s config: %v", err) + logrus.Debugf("unable to parse k0s config: %v", err) return "" } if len(cfg.Spec.WorkerProfiles) > 0 { From a98d1ee3189470d6280eb2eb5730bb23e06cc98f Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Mon, 24 Mar 2025 16:09:43 +0000 Subject: [PATCH 39/48] PR comments --- pkg/config/config.go | 10 +++++----- pkg/config/config_test.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 604ea78c8..834342032 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -39,8 +39,8 @@ func RenderK0sConfig(proxyRegistryDomain string) *k0sconfig.ClusterConfig { return cfg } -// ExtractK0sConfigPatch extracts the k0s config portion of the provided patch. -func ExtractK0sConfigPatch(raw string) (string, error) { +// extractK0sConfigPatch extracts the k0s config portion of the provided patch. +func extractK0sConfigPatch(raw string) (string, error) { type PatchBody struct { Config map[string]interface{} `yaml:"config"` } @@ -62,7 +62,7 @@ func PatchK0sConfig(config *k0sconfig.ClusterConfig, patch string) (*k0sconfig.C if patch == "" { return config, nil } - patch, err := ExtractK0sConfigPatch(patch) + patch, err := extractK0sConfigPatch(patch) if err != nil { return nil, fmt.Errorf("unable to extract k0s config patch: %w", err) } @@ -103,7 +103,7 @@ func InstallFlags(nodeIP string, cfg *k0sconfig.ClusterConfig) []string { "--no-taints", "-c", runtimeconfig.PathToK0sConfig(), } - flags = append(flags, ProfileInstallFlag(cfg)) + flags = append(flags, ProfileInstallFlag()) flags = append(flags, AdditionalInstallFlags(nodeIP)...) flags = append(flags, AdditionalInstallFlagsController()...) return flags @@ -125,7 +125,7 @@ func AdditionalInstallFlagsController() []string { } } -func ProfileInstallFlag(cfg *k0sconfig.ClusterConfig) string { +func ProfileInstallFlag() string { controllerProfile := controllerWorkerProfile() if controllerProfile == "" { return "" diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 7b6e7cc22..ed38cdbf0 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -78,7 +78,7 @@ func Test_extractK0sConfigPatch(t *testing.T) { t.Run(tname, func(t *testing.T) { req := require.New(t) - extracted, err := ExtractK0sConfigPatch(tt.Override) + extracted, err := extractK0sConfigPatch(tt.Override) req.NoError(err) var actual map[string]interface{} From 0ff20de46a208bc2abf3299e325505bf320b1d2f Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Mon, 24 Mar 2025 16:16:30 +0000 Subject: [PATCH 40/48] Remove unused config from other function sigs --- cmd/installer/cli/install.go | 2 +- pkg/config/config.go | 2 +- pkg/k0s/install.go | 4 ++-- pkg/kotsadm/types.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/installer/cli/install.go b/cmd/installer/cli/install.go index 6e1fd859f..50859229c 100644 --- a/cmd/installer/cli/install.go +++ b/cmd/installer/cli/install.go @@ -623,7 +623,7 @@ func installAndStartCluster(ctx context.Context, networkInterface string, airgap } logrus.Debugf("installing k0s") - if err := k0s.Install(networkInterface, cfg); err != nil { + if err := k0s.Install(networkInterface); err != nil { return nil, fmt.Errorf("install cluster: %w", err) } diff --git a/pkg/config/config.go b/pkg/config/config.go index 834342032..3243136de 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -94,7 +94,7 @@ func PatchK0sConfig(config *k0sconfig.ClusterConfig, patch string) (*k0sconfig.C } // InstallFlags returns a list of default flags to be used when bootstrapping a k0s cluster. -func InstallFlags(nodeIP string, cfg *k0sconfig.ClusterConfig) []string { +func InstallFlags(nodeIP string) []string { flags := []string{ "install", "controller", diff --git a/pkg/k0s/install.go b/pkg/k0s/install.go index f79eb5f60..e79634080 100644 --- a/pkg/k0s/install.go +++ b/pkg/k0s/install.go @@ -20,7 +20,7 @@ import ( // Install runs the k0s install command and waits for it to finish. If no configuration // is found one is generated. -func Install(networkInterface string, cfg *k0sv1beta1.ClusterConfig) error { +func Install(networkInterface string) error { ourbin := runtimeconfig.PathToEmbeddedClusterBinary("k0s") hstbin := runtimeconfig.K0sBinaryPath() if err := helpers.MoveFile(ourbin, hstbin); err != nil { @@ -31,7 +31,7 @@ func Install(networkInterface string, cfg *k0sv1beta1.ClusterConfig) error { if err != nil { return fmt.Errorf("unable to find first valid address: %w", err) } - flags := config.InstallFlags(nodeIP, cfg) + flags := config.InstallFlags(nodeIP) if _, err := helpers.RunCommand(hstbin, flags...); err != nil { return fmt.Errorf("unable to install: %w", err) } diff --git a/pkg/kotsadm/types.go b/pkg/kotsadm/types.go index e7ef813e3..4518777a8 100644 --- a/pkg/kotsadm/types.go +++ b/pkg/kotsadm/types.go @@ -21,7 +21,7 @@ type JoinCommandResponse struct { } // extractK0sConfigOverridePatch parses the provided override and returns a dig.Mapping that -// can be then applied on top a k0s configuration file to set both `api` and `storage` spec +// can be then applied on top a k0s configuration file to set `api`, `storage` and `workerProfiles` spec // fields. All other fields in the override are ignored. func (j JoinCommandResponse) extractK0sConfigOverridePatch(data []byte) (dig.Mapping, error) { config := dig.Mapping{} From 05cf0322a376ba7eb8e0e2244e2759a0ebe364a0 Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Mon, 24 Mar 2025 16:30:53 +0000 Subject: [PATCH 41/48] add worker profiles to e2e apps --- .../cluster-config.yaml | 5 +++++ e2e/kots-release-install-legacydr/cluster-config.yaml | 5 +++++ e2e/kots-release-install-stable/cluster-config.yaml | 5 +++++ .../cluster-config.yaml | 5 +++++ e2e/kots-release-upgrade/cluster-config.yaml | 5 +++++ 5 files changed, 25 insertions(+) diff --git a/e2e/kots-release-install-failing-preflights/cluster-config.yaml b/e2e/kots-release-install-failing-preflights/cluster-config.yaml index 4eac429ff..2634327fd 100644 --- a/e2e/kots-release-install-failing-preflights/cluster-config.yaml +++ b/e2e/kots-release-install-failing-preflights/cluster-config.yaml @@ -40,6 +40,11 @@ spec: spec: telemetry: enabled: false + workerProfiles: + - name: ip-forward + values: + allowedUnsafeSysctls: + - net.ipv4.ip_forward extensions: helm: repositories: diff --git a/e2e/kots-release-install-legacydr/cluster-config.yaml b/e2e/kots-release-install-legacydr/cluster-config.yaml index 4eac429ff..2634327fd 100644 --- a/e2e/kots-release-install-legacydr/cluster-config.yaml +++ b/e2e/kots-release-install-legacydr/cluster-config.yaml @@ -40,6 +40,11 @@ spec: spec: telemetry: enabled: false + workerProfiles: + - name: ip-forward + values: + allowedUnsafeSysctls: + - net.ipv4.ip_forward extensions: helm: repositories: diff --git a/e2e/kots-release-install-stable/cluster-config.yaml b/e2e/kots-release-install-stable/cluster-config.yaml index 1e7357761..2b49dfa3e 100644 --- a/e2e/kots-release-install-stable/cluster-config.yaml +++ b/e2e/kots-release-install-stable/cluster-config.yaml @@ -38,6 +38,11 @@ spec: spec: telemetry: enabled: false + workerProfiles: + - name: ip-forward + values: + allowedUnsafeSysctls: + - net.ipv4.ip_forward extensions: helm: repositories: diff --git a/e2e/kots-release-install-warning-preflights/cluster-config.yaml b/e2e/kots-release-install-warning-preflights/cluster-config.yaml index 4eac429ff..2634327fd 100644 --- a/e2e/kots-release-install-warning-preflights/cluster-config.yaml +++ b/e2e/kots-release-install-warning-preflights/cluster-config.yaml @@ -40,6 +40,11 @@ spec: spec: telemetry: enabled: false + workerProfiles: + - name: ip-forward + values: + allowedUnsafeSysctls: + - net.ipv4.ip_forward extensions: helm: repositories: diff --git a/e2e/kots-release-upgrade/cluster-config.yaml b/e2e/kots-release-upgrade/cluster-config.yaml index 46543df5a..6e30d570f 100644 --- a/e2e/kots-release-upgrade/cluster-config.yaml +++ b/e2e/kots-release-upgrade/cluster-config.yaml @@ -46,6 +46,11 @@ spec: spec: telemetry: enabled: false + workerProfiles: + - name: ip-forward + values: + allowedUnsafeSysctls: + - net.ipv4.ip_forward extensions: helm: charts: From 1a4eadc9c6d213e062eb0ae85ff04aefa3ec21ec Mon Sep 17 00:00:00 2001 From: hedge-sparrow Date: Mon, 24 Mar 2025 16:46:47 +0000 Subject: [PATCH 42/48] add check for worker profiles --- e2e/scripts/check-installation-state.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/e2e/scripts/check-installation-state.sh b/e2e/scripts/check-installation-state.sh index cb05a5da6..d37af7039 100755 --- a/e2e/scripts/check-installation-state.sh +++ b/e2e/scripts/check-installation-state.sh @@ -53,6 +53,16 @@ main() { validate_data_dirs fi + # check worker profiles applied correctly + if ! grep -- "--profile=ip-forward" /etc/systemd/system/k0scontroller.service >/dev/null; then + echo "expected worker profile 'ip-forward' not found in k0scontroller.service" + exit 1 + fi + + + + + validate_no_pods_in_crashloop } From b8de1bf436539be4c04e5ebab8c2a0df5d7790cf Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 25 Mar 2025 14:44:19 -0400 Subject: [PATCH 43/48] rework worker profile checking --- e2e/install_test.go | 30 ++++++++++++++++++++++++- e2e/scripts/check-installation-state.sh | 10 +-------- e2e/scripts/check-worker-profile.sh | 11 +++++++++ e2e/scripts/common.sh | 7 ++++++ 4 files changed, 48 insertions(+), 10 deletions(-) create mode 100755 e2e/scripts/check-worker-profile.sh diff --git a/e2e/install_test.go b/e2e/install_test.go index 1a56972a6..ab106d78c 100644 --- a/e2e/install_test.go +++ b/e2e/install_test.go @@ -422,6 +422,13 @@ func TestMultiNodeInstallation(t *testing.T) { if stdout, stderr, err := tc.RunCommandOnNode(node, strings.Split(cmd, " ")); err != nil { t.Fatalf("fail to join node %d as a controller: %v: %s: %s", node, err, stdout, stderr) } + + t.Logf("checking worker profile on controller node %d", node) + line := []string{"check-worker-profile.sh"} + if stdout, stderr, err := tc.RunCommandOnNode(node, line); err != nil { + t.Fatalf("fail to check worker profile on node %d: %v: %s: %s", node, err, stdout, stderr) + } + // XXX If we are too aggressive joining nodes we can see the following error being // thrown by kotsadm on its log (and we get a 500 back): // " @@ -436,6 +443,12 @@ func TestMultiNodeInstallation(t *testing.T) { t.Fatalf("fail to join node 3 to the cluster as a worker: %v: %s: %s", err, stdout, stderr) } + t.Logf("checking worker profile on worker node %d", 3) + line := []string{"check-worker-profile.sh"} + if stdout, stderr, err := tc.RunCommandOnNode(3, line); err != nil { + t.Fatalf("fail to check worker profile on node %d: %v: %s: %s", 3, err, stdout, stderr) + } + // wait for the nodes to report as ready. t.Logf("%s: all nodes joined, waiting for them to be ready", time.Now().Format(time.RFC3339)) stdout, stderr, err = tc.RunCommandOnNode(0, []string{"wait-for-ready-nodes.sh", "4"}) @@ -444,7 +457,7 @@ func TestMultiNodeInstallation(t *testing.T) { } t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339)) - line := []string{"check-installation-state.sh", os.Getenv("SHORT_SHA"), k8sVersion()} + line = []string{"check-installation-state.sh", os.Getenv("SHORT_SHA"), k8sVersion()} if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr) } @@ -2057,6 +2070,11 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) { if _, _, err := tc.RunCommandOnNode(1, strings.Split(command, " ")); err != nil { t.Fatalf("fail to join node 1 to the cluster as a worker: %v", err) } + t.Logf("checking worker profile on worker node %d", 1) + line = []string{"check-worker-profile.sh"} + if stdout, stderr, err := tc.RunCommandOnNode(1, line); err != nil { + t.Fatalf("fail to check worker profile on node %d: %v: %s: %s", 1, err, stdout, stderr) + } // remove the airgap bundle and binary after joining line = []string{"rm", "/assets/release.airgap"} if _, _, err := tc.RunCommandOnNode(1, line); err != nil { @@ -2086,6 +2104,11 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) { if _, _, err := tc.RunCommandOnNode(2, strings.Split(command, " ")); err != nil { t.Fatalf("fail to join node 2 as a controller: %v", err) } + t.Logf("checking worker profile on controller node %d", 2) + line = []string{"check-worker-profile.sh"} + if stdout, stderr, err := tc.RunCommandOnNode(2, line); err != nil { + t.Fatalf("fail to check worker profile on node %d: %v: %s: %s", 2, err, stdout, stderr) + } // remove the airgap bundle and binary after joining line = []string{"rm", "/assets/release.airgap"} if _, _, err := tc.RunCommandOnNode(2, line); err != nil { @@ -2113,6 +2136,11 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) { if _, _, err := tc.RunCommandOnNode(3, line); err != nil { t.Fatalf("fail to join node 3 as a controller in ha mode: %v", err) } + t.Logf("checking worker profile on controller node %d", 3) + line = []string{"check-worker-profile.sh"} + if stdout, stderr, err := tc.RunCommandOnNode(3, line); err != nil { + t.Fatalf("fail to check worker profile on node %d: %v: %s: %s", 3, err, stdout, stderr) + } // remove the airgap bundle and binary after joining line = []string{"rm", "/assets/release.airgap"} if _, _, err := tc.RunCommandOnNode(3, line); err != nil { diff --git a/e2e/scripts/check-installation-state.sh b/e2e/scripts/check-installation-state.sh index d37af7039..56281b2c2 100755 --- a/e2e/scripts/check-installation-state.sh +++ b/e2e/scripts/check-installation-state.sh @@ -51,18 +51,10 @@ main() { # if this is the current version in CI if echo "$version" | grep -qvE "(pre-minio-removal|1.8.0-k8s)" ; then validate_data_dirs - fi - # check worker profiles applied correctly - if ! grep -- "--profile=ip-forward" /etc/systemd/system/k0scontroller.service >/dev/null; then - echo "expected worker profile 'ip-forward' not found in k0scontroller.service" - exit 1 + validate_worker_profile fi - - - - validate_no_pods_in_crashloop } diff --git a/e2e/scripts/check-worker-profile.sh b/e2e/scripts/check-worker-profile.sh new file mode 100755 index 000000000..04f722458 --- /dev/null +++ b/e2e/scripts/check-worker-profile.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euox pipefail + +DIR=/usr/local/bin +. $DIR/common.sh + +main() { + validate_worker_profile +} + +main "$@" diff --git a/e2e/scripts/common.sh b/e2e/scripts/common.sh index 9c4049aff..187b7cd14 100755 --- a/e2e/scripts/common.sh +++ b/e2e/scripts/common.sh @@ -462,3 +462,10 @@ validate_no_pods_in_crashloop() { exit 1 fi } + +validate_worker_profile() { + if ! grep -- "--profile=ip-forward" /etc/systemd/system/k0scontroller.service >/dev/null; then + echo "expected worker profile 'ip-forward' not found in k0scontroller.service" + exit 1 + fi +} From 83cd7eb16283373c7222077fd7f89f7f18964db6 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 25 Mar 2025 15:07:23 -0400 Subject: [PATCH 44/48] previous-stable is not the current version --- e2e/scripts/check-installation-state.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/scripts/check-installation-state.sh b/e2e/scripts/check-installation-state.sh index 56281b2c2..6f587b9af 100755 --- a/e2e/scripts/check-installation-state.sh +++ b/e2e/scripts/check-installation-state.sh @@ -49,7 +49,7 @@ main() { fi # if this is the current version in CI - if echo "$version" | grep -qvE "(pre-minio-removal|1.8.0-k8s)" ; then + if echo "$version" | grep -qvE "(pre-minio-removal|1.8.0-k8s|previous-stable)" ; then validate_data_dirs validate_worker_profile From a19a6d455e88d1f4db66f1a6c7da00d567af999d Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 25 Mar 2025 15:36:45 -0400 Subject: [PATCH 45/48] fix check for profile on worker nodes --- e2e/scripts/common.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/e2e/scripts/common.sh b/e2e/scripts/common.sh index 187b7cd14..676554c28 100755 --- a/e2e/scripts/common.sh +++ b/e2e/scripts/common.sh @@ -464,8 +464,19 @@ validate_no_pods_in_crashloop() { } validate_worker_profile() { - if ! grep -- "--profile=ip-forward" /etc/systemd/system/k0scontroller.service >/dev/null; then - echo "expected worker profile 'ip-forward' not found in k0scontroller.service" + # if /etc/systemd/system/k0scontroller.service exists, check it - otherwise check /etc/systemd/system/k0sworker.service + if [ -f /etc/systemd/system/k0scontroller.service ]; then + if ! grep -- "--profile=ip-forward" /etc/systemd/system/k0scontroller.service >/dev/null; then + echo "expected worker profile 'ip-forward' not found in k0scontroller.service" + exit 1 + fi + elif [ -f /etc/systemd/system/k0sworker.service ]; then + if ! grep -- "--profile=ip-forward" /etc/systemd/system/k0sworker.service >/dev/null; then + echo "expected worker profile 'ip-forward' not found in k0sworker.service" + exit 1 + fi + else + echo "expected k0scontroller.service or k0sworker.service not found" exit 1 fi } From 25dcebd2ed920365ebc48dc2009a122c534d872f Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 25 Mar 2025 16:38:24 -0400 Subject: [PATCH 46/48] remove worker profile check from check-installation-state due to old-version-upgrade tests --- e2e/install_test.go | 13 ++++++++++++- e2e/scripts/check-installation-state.sh | 2 -- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/e2e/install_test.go b/e2e/install_test.go index ab106d78c..6a7fb5907 100644 --- a/e2e/install_test.go +++ b/e2e/install_test.go @@ -384,6 +384,11 @@ func TestMultiNodeInstallation(t *testing.T) { if stdout, stderr, err := tc.RunCommandOnNode(0, []string{"single-node-install.sh", "ui", os.Getenv("SHORT_SHA")}); err != nil { t.Fatalf("fail to install embedded-cluster on node 0: %v: %s: %s", err, stdout, stderr) } + t.Logf("checking worker profile on controller node %d", 0) + line := []string{"check-worker-profile.sh"} + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { + t.Fatalf("fail to check worker profile on node %d: %v: %s: %s", 0, err, stdout, stderr) + } if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil { t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr) @@ -444,7 +449,7 @@ func TestMultiNodeInstallation(t *testing.T) { } t.Logf("checking worker profile on worker node %d", 3) - line := []string{"check-worker-profile.sh"} + line = []string{"check-worker-profile.sh"} if stdout, stderr, err := tc.RunCommandOnNode(3, line); err != nil { t.Fatalf("fail to check worker profile on node %d: %v: %s: %s", 3, err, stdout, stderr) } @@ -2031,6 +2036,12 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) { if _, _, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err) } + t.Logf("checking worker profile on controller node %d", 0) + line = []string{"check-worker-profile.sh"} + if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil { + t.Fatalf("fail to check worker profile on node %d: %v: %s: %s", 0, err, stdout, stderr) + } + // remove artifacts after installation to save space line = []string{"rm", "/assets/release.airgap"} if _, _, err := tc.RunCommandOnNode(0, line); err != nil { diff --git a/e2e/scripts/check-installation-state.sh b/e2e/scripts/check-installation-state.sh index 6f587b9af..1937b1711 100755 --- a/e2e/scripts/check-installation-state.sh +++ b/e2e/scripts/check-installation-state.sh @@ -51,8 +51,6 @@ main() { # if this is the current version in CI if echo "$version" | grep -qvE "(pre-minio-removal|1.8.0-k8s|previous-stable)" ; then validate_data_dirs - - validate_worker_profile fi validate_no_pods_in_crashloop From 51b3ee775f08f8029c9bea18649f4d20ac66e83b Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 25 Mar 2025 18:09:40 -0400 Subject: [PATCH 47/48] pass errors when relevant --- cmd/installer/cli/join.go | 17 +++++++++-------- pkg/config/config.go | 30 +++++++++++++++++------------- pkg/k0s/install.go | 5 ++++- 3 files changed, 30 insertions(+), 22 deletions(-) diff --git a/cmd/installer/cli/join.go b/cmd/installer/cli/join.go index 1a24b5ccf..c1e1ef7cc 100644 --- a/cmd/installer/cli/join.go +++ b/cmd/installer/cli/join.go @@ -341,7 +341,10 @@ func installAndJoinCluster(ctx context.Context, jcmd *kotsadm.JoinCommandRespons return fmt.Errorf("unable to apply configuration overrides: %w", err) } - profile := getFirstDefinedProfile() + profile, err := getFirstDefinedProfile() + if err != nil { + return fmt.Errorf("unable to get first defined profile: %w", err) + } logrus.Debugf("joining node to cluster") if err := runK0sInstallCommand(flags.networkInterface, jcmd.K0sJoinCommand, profile); err != nil { @@ -462,22 +465,20 @@ func applyJoinConfigurationOverrides(jcmd *kotsadm.JoinCommandResponse) error { return nil } -func getFirstDefinedProfile() string { +func getFirstDefinedProfile() (string, error) { k0scfg, err := os.Open(runtimeconfig.PathToK0sConfig()) if err != nil { - logrus.Debugf("unable to open k0s config: %v", err) - return "" + return "", fmt.Errorf("unable to open k0s config: %w", err) } defer k0scfg.Close() cfg, err := k0sconfig.ConfigFromReader(k0scfg) if err != nil { - logrus.Debugf("unable to parse k0s config: %v", err) - return "" + return "", fmt.Errorf("unable to parse k0s config: %w", err) } if len(cfg.Spec.WorkerProfiles) > 0 { - return cfg.Spec.WorkerProfiles[0].Name + return cfg.Spec.WorkerProfiles[0].Name, nil } - return "" + return "", nil } // runK0sInstallCommand runs the k0s install command as provided by the kots diff --git a/pkg/config/config.go b/pkg/config/config.go index 3243136de..f5a8306df 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -94,7 +94,7 @@ func PatchK0sConfig(config *k0sconfig.ClusterConfig, patch string) (*k0sconfig.C } // InstallFlags returns a list of default flags to be used when bootstrapping a k0s cluster. -func InstallFlags(nodeIP string) []string { +func InstallFlags(nodeIP string) ([]string, error) { flags := []string{ "install", "controller", @@ -103,10 +103,14 @@ func InstallFlags(nodeIP string) []string { "--no-taints", "-c", runtimeconfig.PathToK0sConfig(), } - flags = append(flags, ProfileInstallFlag()) + profile, err := ProfileInstallFlag() + if err != nil { + return nil, fmt.Errorf("unable to get profile install flag: %w", err) + } + flags = append(flags, profile) flags = append(flags, AdditionalInstallFlags(nodeIP)...) flags = append(flags, AdditionalInstallFlagsController()...) - return flags + return flags, nil } func AdditionalInstallFlags(nodeIP string) []string { @@ -125,12 +129,12 @@ func AdditionalInstallFlagsController() []string { } } -func ProfileInstallFlag() string { - controllerProfile := controllerWorkerProfile() - if controllerProfile == "" { - return "" +func ProfileInstallFlag() (string, error) { + controllerProfile, err := controllerWorkerProfile() + if err != nil { + return "", fmt.Errorf("unable to get controller worker profile: %w", err) } - return "--profile=" + controllerProfile + return "--profile=" + controllerProfile, nil } // nodeLabels return a slice of string with labels (key=value format) for the node where we @@ -171,23 +175,23 @@ func additionalControllerLabels() map[string]string { return map[string]string{} } -func controllerWorkerProfile() string { +func controllerWorkerProfile() (string, error) { // Read the k0s config file data, err := os.ReadFile(runtimeconfig.PathToK0sConfig()) if err != nil { - return "" + return "", fmt.Errorf("unable to read k0s config: %w", err) } var cfg k0sconfig.ClusterConfig if err := k8syaml.Unmarshal(data, &cfg); err != nil { - return "" + return "", fmt.Errorf("unable to unmarshal k0s config: %w", err) } // Return the first worker profile name if any exist if len(cfg.Spec.WorkerProfiles) > 0 { - return cfg.Spec.WorkerProfiles[0].Name + return cfg.Spec.WorkerProfiles[0].Name, nil } - return "" + return "", nil } func AdditionalCharts() []embeddedclusterv1beta1.Chart { diff --git a/pkg/k0s/install.go b/pkg/k0s/install.go index e79634080..28078bfa6 100644 --- a/pkg/k0s/install.go +++ b/pkg/k0s/install.go @@ -31,7 +31,10 @@ func Install(networkInterface string) error { if err != nil { return fmt.Errorf("unable to find first valid address: %w", err) } - flags := config.InstallFlags(nodeIP) + flags, err := config.InstallFlags(nodeIP) + if err != nil { + return fmt.Errorf("unable to get install flags: %w", err) + } if _, err := helpers.RunCommand(hstbin, flags...); err != nil { return fmt.Errorf("unable to install: %w", err) } From 0233f9a70d89c21a9b49564a8676b981578f9e8c Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Tue, 25 Mar 2025 18:31:53 -0400 Subject: [PATCH 48/48] fix dryrun join test, add more nil checking --- pkg/helpers/k0s.go | 24 ++++++++++++++++++++++++ tests/dryrun/join_test.go | 15 ++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/pkg/helpers/k0s.go b/pkg/helpers/k0s.go index 77b87ecf1..b8a39c408 100644 --- a/pkg/helpers/k0s.go +++ b/pkg/helpers/k0s.go @@ -22,6 +22,30 @@ func K0sClusterConfigTo129Compat(clusterConfig *k0sv1beta1.ClusterConfig) (*unst return nil, fmt.Errorf("convert to unstructured: %w", err) } unst := obj.UnstructuredContent() + + // check the entire spec path before attempting to access "charts" + if unst["spec"] == nil { + return obj, nil + } + if _, ok := unst["spec"].(map[string]interface{}); !ok { + return obj, nil + } + if _, ok := unst["spec"].(map[string]interface{})["extensions"]; !ok { + return obj, nil + } + if _, ok := unst["spec"].(map[string]interface{})["extensions"].(map[string]interface{}); !ok { + return obj, nil + } + if _, ok := unst["spec"].(map[string]interface{})["extensions"].(map[string]interface{})["helm"]; !ok { + return obj, nil + } + if _, ok := unst["spec"].(map[string]interface{})["extensions"].(map[string]interface{})["helm"].(map[string]interface{}); !ok { + return obj, nil + } + if _, ok := unst["spec"].(map[string]interface{})["extensions"].(map[string]interface{})["helm"].(map[string]interface{})["charts"]; !ok { + return obj, nil + } + charts, ok := unst["spec"].(map[string]interface{})["extensions"].(map[string]interface{})["helm"].(map[string]interface{})["charts"].([]interface{}) if !ok { return obj, nil diff --git a/tests/dryrun/join_test.go b/tests/dryrun/join_test.go index ed9e8d582..b8eec7e3e 100644 --- a/tests/dryrun/join_test.go +++ b/tests/dryrun/join_test.go @@ -191,7 +191,20 @@ func TestJoinWorkerNode(t *testing.T) { InstallationSpec: ecv1beta1.InstallationSpec{ ClusterID: clusterID.String(), Config: &ecv1beta1.ConfigSpec{ - UnsupportedOverrides: ecv1beta1.UnsupportedOverrides{}, + UnsupportedOverrides: ecv1beta1.UnsupportedOverrides{ + K0s: ` +config: + metadata: + name: foo + spec: + telemetry: + enabled: false + workerProfiles: + - name: ip-forward + values: + allowedUnsafeSysctls: + - net.ipv4.ip_forward`, + }, }, }, }