ignition: add vpc ipv6 cidr to installconfig if needed

The installconfig in the cluster-config ConfigMap needs to have the
Ipv6 CIDR of the VPC in the case of full IPI.

Author: tthvo

pkg/infrastructure/aws/clusterapi/ignition.go

@@ -13,20 +13,53 @@ import (
 	capa "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
 	k8sClient "sigs.k8s.io/controller-runtime/pkg/client"
 
+	machinev1 "github.com/openshift/api/machineconfiguration/v1"
 	"github.com/openshift/installer/pkg/asset/manifests/capiutils"
 	"github.com/openshift/installer/pkg/infrastructure/clusterapi"
+	"github.com/openshift/installer/pkg/ipnet"
+	"github.com/openshift/installer/pkg/types"
 	awstypes "github.com/openshift/installer/pkg/types/aws"
 	"github.com/openshift/installer/pkg/types/dns"
 )
 
 func editIgnition(ctx context.Context, in clusterapi.IgnitionInput) (*clusterapi.IgnitionOutput, error) {
-	if in.InstallConfig.Config.AWS.UserProvisionedDNS != dns.UserProvisionedDNSEnabled {
-		return &clusterapi.IgnitionOutput{
-			UpdatedBootstrapIgn: in.BootstrapIgnData,
-			UpdatedMasterIgn:    in.MasterIgnData,
-			UpdatedWorkerIgn:    in.WorkerIgnData}, nil
+	// To be revised: The IgnitionOutput and IgnitionInput can be chained in a series of editIgnition calls.
+	// To update output with latest ignition output, use (*IgnitionOutput).Set(*IgnitionOutput).
+	// To update input with latest ignition output (to use as input for another editIgnition call), use (*IgnitionInput).SetIgnFromOutput(*IgnitionOutput).
+	ignOutput := &clusterapi.IgnitionOutput{
+		UpdatedBootstrapIgn: in.BootstrapIgnData,
+		UpdatedMasterIgn:    in.MasterIgnData,
+		UpdatedWorkerIgn:    in.WorkerIgnData,
 	}
 
+	// Dualstack mode
+	if in.InstallConfig.Config.IsDualStackInfra() {
+		// If there is no IPv6 machine network in the install-config, that means AWS will assign a IPv6 GUA range during
+		// infrastructure provisioning. Thus, after VPC is ready, we need to populate the install config with the VPC IPv6 CIDR.
+		if len(capiutils.MachineCIDRsFromInstallConfig(in.InstallConfig).IPv6Nets()) == 0 {
+			dualStackIgnOut, err := editIgnitionForDualStack(ctx, in)
+			if err != nil {
+				return ignOutput, err
+			}
+			ignOutput.Set(dualStackIgnOut)
+			in.SetIgnFromOutput(ignOutput)
+		}
+	}
+
+	// Custom DNS
+	if in.InstallConfig.Config.AWS.UserProvisionedDNS == dns.UserProvisionedDNSEnabled {
+		customDNSIgnOut, err := editIgnitionForCustomDNS(ctx, in)
+		if err != nil {
+			return ignOutput, err
+		}
+		ignOutput.Set(customDNSIgnOut)
+		in.SetIgnFromOutput(ignOutput)
+	}
+
+	return ignOutput, nil
+}
+
+func editIgnitionForCustomDNS(ctx context.Context, in clusterapi.IgnitionInput) (*clusterapi.IgnitionOutput, error) {
 	awsCluster := &capa.AWSCluster{}
 	key := k8sClient.ObjectKey{
 		Name:      in.InfraID,
@@ -89,5 +122,36 @@ func editIgnition(ctx context.Context, in clusterapi.IgnitionInput) (*clusterapi
 		publicIPAddresses = privateIPAddresses
 	}
 	logrus.Debugf("AWS: Editing Ignition files to start in-cluster DNS when UserProvisionedDNS is enabled")
-	return clusterapi.EditIgnition(in, awstypes.Name, publicIPAddresses, privateIPAddresses)
+	return clusterapi.EditIgnitionForCustomDNS(in, awstypes.Name, publicIPAddresses, privateIPAddresses)
+}
+
+func editIgnitionForDualStack(ctx context.Context, in clusterapi.IgnitionInput) (*clusterapi.IgnitionOutput, error) {
+	awsCluster := &capa.AWSCluster{}
+	key := k8sClient.ObjectKey{
+		Name:      in.InfraID,
+		Namespace: capiutils.Namespace,
+	}
+	if err := in.Client.Get(ctx, key, awsCluster); err != nil {
+		return nil, fmt.Errorf("failed to get AWSCluster: %w", err)
+	}
+
+	machineNetworks := in.InstallConfig.Config.MachineNetwork
+	// Assumption: When IPv6 is enabled, the AWSCluster IPv6 block should be non-nil
+	// And the VPC IPv6 CIDR is valid (i.e. set by CAPA)
+	vpcIPv6CIDR := awsCluster.Spec.NetworkSpec.VPC.IPv6.CidrBlock
+
+	ic := in.InstallConfig.Config
+	ipv6Entry := []types.MachineNetworkEntry{
+		{
+			CIDR: *ipnet.MustParseCIDR(vpcIPv6CIDR),
+		},
+	}
+	if ic.InfraStack() == machinev1.IPFamiliesDualStackIPv6Primary {
+		machineNetworks = append(ipv6Entry, machineNetworks...)
+	} else {
+		machineNetworks = append(machineNetworks, ipv6Entry...)
+	}
+
+	logrus.Debugf("AWS: Editing Ignition files to add VPC IPv6 CIDR to the machine")
+	return clusterapi.EditIgnitionForDualStack(in, awstypes.Name, machineNetworks)
 }

pkg/infrastructure/clusterapi/ignition.go

@@ -20,6 +20,7 @@ import (
 	"github.com/openshift/installer/pkg/asset/lbconfig"
 	"github.com/openshift/installer/pkg/asset/machines"
 	"github.com/openshift/installer/pkg/asset/tls"
+	"github.com/openshift/installer/pkg/types"
 	awstypes "github.com/openshift/installer/pkg/types/aws"
 	gcptypes "github.com/openshift/installer/pkg/types/gcp"
 )
@@ -31,20 +32,24 @@ const (
 	mcsCertFile            = "/opt/openshift/tls/machine-config-server.crt"
 	masterUserDataFile     = "/opt/openshift/openshift/99_openshift-cluster-api_master-user-data-secret.yaml"
 	workerUserDataFile     = "/opt/openshift/openshift/99_openshift-cluster-api_worker-user-data-secret.yaml"
+	clusterConfigDataFile  = "/opt/openshift/manifests/cluster-config.yaml"
 
 	// header is the string that precedes the encoded data in the ignition data.
 	// The data must be replaced before decoding the string, and the string must be
 	// prepended to the encoded data.
 	header = "data:text/plain;charset=utf-8;base64,"
 
+	// The key in the cluster-config-v1 ConfigMap to extract the install-config.
+	clusterConfigCMKey = "install-config"
+
 	masterRole = "master"
 	workerRole = "worker"
 )
 
-// EditIgnition attempts to edit the contents of the bootstrap ignition when the user has selected
+// EditIgnitionForCustomDNS attempts to edit the contents of the bootstrap ignition when the user has selected
 // a custom DNS configuration. Find the public and private load balancer addresses and fill in the
 // infrastructure file within the ignition struct.
-func EditIgnition(in IgnitionInput, platform string, publicIPAddresses, privateIPAddresses []string) (*IgnitionOutput, error) {
+func EditIgnitionForCustomDNS(in IgnitionInput, platform string, publicIPAddresses, privateIPAddresses []string) (*IgnitionOutput, error) {
 	ignData := &igntypes.Config{}
 	err := json.Unmarshal(in.BootstrapIgnData, ignData)
 	if err != nil {
@@ -287,3 +292,76 @@ func updateUserDataSecret(in IgnitionInput, role string, config *igntypes.Config
 	}
 	return nil
 }
+
+// EditIgnitionForDualStack attempts to edit the contents of the bootstrap ignition when the cluster is in dualstack.
+// This addes the missing IPv6 manchine network for platform where IPv6 CIDRs are not known at install time, for example, AWS.
+func EditIgnitionForDualStack(in IgnitionInput, platform string, machineNetworks []types.MachineNetworkEntry) (*IgnitionOutput, error) {
+	ignData := &igntypes.Config{}
+	err := json.Unmarshal(in.BootstrapIgnData, ignData)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal bootstrap ignition: %w", err)
+	}
+
+	err = updateMachineNetworks(in, ignData, machineNetworks)
+	if err != nil {
+		return nil, fmt.Errorf("failed to add VPC IPv6 CIDR to ignition config: %w", err)
+	}
+	logrus.Debugf("Successfully added VPC IPv6 CIDR to the install-config machine networks")
+
+	editedIgnBytes, err := json.Marshal(ignData)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert ignition data to json: %w", err)
+	}
+	logrus.Debugf("Successfully updated bootstrap ignition with updated manifests")
+
+	return &IgnitionOutput{
+		UpdatedBootstrapIgn: editedIgnBytes,
+		UpdatedMasterIgn:    in.MasterIgnData,
+		UpdatedWorkerIgn:    in.WorkerIgnData,
+	}, nil
+}
+
+func updateMachineNetworks(in IgnitionInput, config *igntypes.Config, machineNetworks []types.MachineNetworkEntry) error {
+	for i, fileData := range config.Storage.Files {
+		if fileData.Path == clusterConfigDataFile {
+			contents := strings.Split(*config.Storage.Files[i].Contents.Source, ",")
+			rawDecodedText, err := base64.StdEncoding.DecodeString(contents[1])
+			if err != nil {
+				return fmt.Errorf("failed to decode contents of ignition file: %w", err)
+			}
+
+			configCM := &corev1.ConfigMap{}
+			if err := yaml.Unmarshal(rawDecodedText, configCM); err != nil {
+				return fmt.Errorf("failed to unmarshal cluster-config ConfigMap: %w", err)
+			}
+
+			installConfig := &types.InstallConfig{}
+			if err := yaml.Unmarshal([]byte(configCM.Data[clusterConfigCMKey]), installConfig); err != nil {
+				return fmt.Errorf("failed to unmarshal install-config content: %w", err)
+			}
+
+			// Update the machine network field
+			installConfig.MachineNetwork = machineNetworks
+
+			// Convert the installconfig back to string and save it to the configmap
+			icContents, err := yaml.Marshal(installConfig)
+			if err != nil {
+				return fmt.Errorf("failed to marshal install-config: %w", err)
+			}
+			configCM.Data[clusterConfigCMKey] = string(icContents)
+
+			// convert the infrastructure back to an encoded string
+			configCMContents, err := yaml.Marshal(configCM)
+			if err != nil {
+				return fmt.Errorf("failed to marshal cluster-config ConfigMap: %w", err)
+			}
+
+			encoded := fmt.Sprintf("%s%s", header, base64.StdEncoding.EncodeToString(configCMContents))
+			// replace the contents with the edited information
+			config.Storage.Files[i].Contents.Source = &encoded
+
+			break
+		}
+	}
+	return nil
+}

pkg/infrastructure/clusterapi/types.go

@@ -72,13 +72,35 @@ type IgnitionInput struct {
 	RootCA           *tls.RootCA
 }
 
+// SetIgnFromOutput set the ignition data for input from an output.
+// This is used to chain multiple ignition editions.
+func (ignIn *IgnitionInput) SetIgnFromOutput(outIgn *IgnitionOutput) {
+	if ignIn == nil || outIgn == nil {
+		return
+	}
+	ignIn.BootstrapIgnData = outIgn.UpdatedBootstrapIgn
+	ignIn.MasterIgnData = outIgn.UpdatedMasterIgn
+	ignIn.WorkerIgnData = outIgn.UpdatedWorkerIgn
+}
+
 // IgnitionOutput collects updated Ignition Data for Bootstrap, Master and Worker nodes.
 type IgnitionOutput struct {
 	UpdatedBootstrapIgn []byte
 	UpdatedMasterIgn    []byte
 	UpdatedWorkerIgn    []byte
 }
 
+// Set set the ignition data from another output.
+// This is used for chained multiple ignition editions.
+func (ignOut *IgnitionOutput) Set(other *IgnitionOutput) {
+	if ignOut == nil || other == nil {
+		return
+	}
+	ignOut.UpdatedBootstrapIgn = other.UpdatedBootstrapIgn
+	ignOut.UpdatedMasterIgn = other.UpdatedMasterIgn
+	ignOut.UpdatedWorkerIgn = other.UpdatedWorkerIgn
+}
+
 // InfraReadyProvider defines the InfraReady hook, which is
 // called after the initial infrastructure manifests have been created
 // and InfrastructureReady == true on the cluster status, and before

pkg/infrastructure/gcp/clusterapi/ignition.go

@@ -70,5 +70,5 @@ func editIgnition(ctx context.Context, in clusterapi.IgnitionInput) (*clusterapi
 	}
 
 	logrus.Debugf("GCP: Editing Ignition files to start in-cluster DNS when UserProvisionedDNS is enabled")
-	return clusterapi.EditIgnition(in, gcp.Name, []string{computeAddress}, []string{computeIntAddress})
+	return clusterapi.EditIgnitionForCustomDNS(in, gcp.Name, []string{computeAddress}, []string{computeIntAddress})
 }