From 05d864a1dcfaa550e9ae3e99c21fa2b79e95dbe6 Mon Sep 17 00:00:00 2001 From: Jon Huhn Date: Tue, 23 May 2023 11:07:14 -0500 Subject: [PATCH] update managed cluster doc --- .markdownlinkcheck.json | 7 +- api/v1beta1/azuremanagedcluster_types.go | 1 + api/v1beta1/azuremanagedcontrolplane_types.go | 38 +- api/v1beta1/azuremanagedmachinepool_types.go | 104 +++- ...cluster.x-k8s.io_azuremanagedclusters.yaml | 3 +- ...er.x-k8s.io_azuremanagedcontrolplanes.yaml | 32 +- ...ter.x-k8s.io_azuremanagedmachinepools.yaml | 170 +++--- docs/book/src/topics/managedcluster.md | 499 +----------------- 8 files changed, 264 insertions(+), 590 deletions(-) diff --git a/.markdownlinkcheck.json b/.markdownlinkcheck.json index 3e808e1bd58..353ac60c8a1 100644 --- a/.markdownlinkcheck.json +++ b/.markdownlinkcheck.json @@ -1,7 +1,8 @@ { - "ignorePatterns": [{ - "pattern": "^https://calendar.google.com/calendar" - }], + "ignorePatterns": [ + { "pattern": "^https://calendar.google.com/calendar" }, + { "pattern": "^../reference/" } + ], "httpHeaders": [{ "comment": "Workaround as suggested here: https://github.com/tcort/markdown-link-check/issues/201", "urls": ["https://docs.github.com/"], diff --git a/api/v1beta1/azuremanagedcluster_types.go b/api/v1beta1/azuremanagedcluster_types.go index 3433c471eac..77442a5744a 100644 --- a/api/v1beta1/azuremanagedcluster_types.go +++ b/api/v1beta1/azuremanagedcluster_types.go @@ -24,6 +24,7 @@ import ( // AzureManagedClusterSpec defines the desired state of AzureManagedCluster. type AzureManagedClusterSpec struct { // ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. + // Immutable, populated by the AKS API at create. // +optional ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"` } diff --git a/api/v1beta1/azuremanagedcontrolplane_types.go b/api/v1beta1/azuremanagedcontrolplane_types.go index 82e159040b3..a144886c2af 100644 --- a/api/v1beta1/azuremanagedcontrolplane_types.go +++ b/api/v1beta1/azuremanagedcontrolplane_types.go @@ -55,26 +55,32 @@ type AzureManagedControlPlaneSpec struct { Version string `json:"version"` // ResourceGroupName is the name of the Azure resource group for this AKS Cluster. + // Immutable. ResourceGroupName string `json:"resourceGroupName"` // NodeResourceGroupName is the name of the resource group // containing cluster IaaS resources. Will be populated to default // in webhook. + // Immutable. // +optional NodeResourceGroupName string `json:"nodeResourceGroupName,omitempty"` // VirtualNetwork describes the vnet for the AKS cluster. Will be created if it does not exist. + // Immutable except for `subnet`. // +optional VirtualNetwork ManagedControlPlaneVirtualNetwork `json:"virtualNetwork,omitempty"` // SubscriptionID is the GUID of the Azure subscription to hold this cluster. + // Immutable. // +optional SubscriptionID string `json:"subscriptionID,omitempty"` // Location is a string matching one of the canonical Azure region names. Examples: "westus2", "eastus". + // Immutable. Location string `json:"location"` // ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. + // Immutable, populated by the AKS API at create. // +optional ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint,omitempty"` @@ -84,29 +90,37 @@ type AzureManagedControlPlaneSpec struct { AdditionalTags Tags `json:"additionalTags,omitempty"` // NetworkPlugin used for building Kubernetes network. + // Allowed values are "azure", "kubenet". + // Immutable. // +kubebuilder:validation:Enum=azure;kubenet // +optional NetworkPlugin *string `json:"networkPlugin,omitempty"` // NetworkPolicy used for building Kubernetes network. + // Allowed values are "azure", "calico". + // Immutable. // +kubebuilder:validation:Enum=azure;calico // +optional NetworkPolicy *string `json:"networkPolicy,omitempty"` // Outbound configuration used by Nodes. + // Immutable. // +kubebuilder:validation:Enum=loadBalancer;managedNATGateway;userAssignedNATGateway;userDefinedRouting // +optional OutboundType *ManagedControlPlaneOutboundType `json:"outboundType,omitempty"` // SSHPublicKey is a string literal containing an ssh public key base64 encoded. + // Immutable. SSHPublicKey string `json:"sshPublicKey"` // DNSServiceIP is an IP address assigned to the Kubernetes DNS service. // It must be within the Kubernetes service address range specified in serviceCidr. + // Immutable. // +optional DNSServiceIP *string `json:"dnsServiceIP,omitempty"` // LoadBalancerSKU is the SKU of the loadBalancer to be provisioned. + // Immutable. // +kubebuilder:validation:Enum=Basic;Standard // +optional LoadBalancerSKU *string `json:"loadBalancerSKU,omitempty"` @@ -132,6 +146,7 @@ type AzureManagedControlPlaneSpec struct { LoadBalancerProfile *LoadBalancerProfile `json:"loadBalancerProfile,omitempty"` // APIServerAccessProfile is the access profile for AKS API server. + // Immutable except for `authorizedIPRanges`. // +optional APIServerAccessProfile *APIServerAccessProfile `json:"apiServerAccessProfile,omitempty"` @@ -149,6 +164,9 @@ type AzureManagedControlPlaneSpec struct { } // AADProfile - AAD integration managed by AKS. +// See also [AKS doc]. +// +// [AKS doc]: https://learn.microsoft.com/azure/aks/managed-aad type AADProfile struct { // Managed - Whether to enable managed AAD. // +kubebuilder:validation:Required @@ -190,12 +208,11 @@ type AKSSku struct { } // LoadBalancerProfile - Profile of the cluster load balancer. +// At most one of `managedOutboundIPs`, `outboundIPPrefixes`, or `outboundIPs` may be specified. +// See also [AKS doc]. +// +// [AKS doc]: https://learn.microsoft.com/azure/aks/load-balancer-standard type LoadBalancerProfile struct { - // Load balancer profile must specify at most one of ManagedOutboundIPs, OutboundIPPrefixes and OutboundIPs. - // By default the AKS cluster automatically creates a public IP in the AKS-managed infrastructure resource group and assigns it to the load balancer outbound pool. - // Alternatively, you can assign your own custom public IP or public IP prefix at cluster creation time. - // See https://docs.microsoft.com/en-us/azure/aks/load-balancer-standard#provide-your-own-outbound-public-ips-or-prefixes - // ManagedOutboundIPs - Desired managed outbound IPs for the cluster load balancer. // +optional ManagedOutboundIPs *int32 `json:"managedOutboundIPs,omitempty"` @@ -217,7 +234,10 @@ type LoadBalancerProfile struct { IdleTimeoutInMinutes *int32 `json:"idleTimeoutInMinutes,omitempty"` } -// APIServerAccessProfile - access profile for AKS API server. +// APIServerAccessProfile tunes the accessibility of the cluster's control plane. +// See also [AKS doc]. +// +// [AKS doc]: https://learn.microsoft.com/azure/aks/api-server-authorized-ip-ranges type APIServerAccessProfile struct { // AuthorizedIPRanges - Authorized IP Ranges to kubernetes API server. // +optional @@ -238,6 +258,7 @@ type APIServerAccessProfile struct { type ManagedControlPlaneVirtualNetwork struct { Name string `json:"name"` CIDRBlock string `json:"cidrBlock"` + // Immutable except for `serviceEndpoints`. // +optional Subnet ManagedControlPlaneSubnet `json:"subnet,omitempty"` // ResourceGroup is the name of the Azure resource group for the VNet and Subnet. @@ -282,7 +303,10 @@ type AzureManagedControlPlaneStatus struct { } // AutoScalerProfile parameters to be applied to the cluster-autoscaler. -// See the [FAQ](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#what-are-the-parameters-to-ca) for more details about each parameter. +// See also [AKS doc], [K8s doc]. +// +// [AKS doc]: https://learn.microsoft.com/azure/aks/cluster-autoscaler#use-the-cluster-autoscaler-profile +// [K8s doc]: https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#what-are-the-parameters-to-ca type AutoScalerProfile struct { // BalanceSimilarNodeGroups - Valid values are 'true' and 'false'. The default is false. // +kubebuilder:validation:Enum="true";"false" diff --git a/api/v1beta1/azuremanagedmachinepool_types.go b/api/v1beta1/azuremanagedmachinepool_types.go index 4af113de91a..48e9499ebc7 100644 --- a/api/v1beta1/azuremanagedmachinepool_types.go +++ b/api/v1beta1/azuremanagedmachinepool_types.go @@ -93,7 +93,11 @@ const ( TransparentHugePageOptionDeferMadvise TransparentHugePageOption = "defer+madvise" ) -// KubeletConfig defines the set of kubelet configurations for nodes in pools. +// KubeletConfig defines the supported subset of kubelet configurations for nodes in pools. +// See also [AKS doc], [K8s doc]. +// +// [AKS doc]: https://learn.microsoft.com/azure/aks/custom-node-configuration +// [K8s doc]: https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/ type KubeletConfig struct { // CPUManagerPolicy - CPU Manager policy to use. // +kubebuilder:validation:Enum=none;static @@ -103,14 +107,17 @@ type KubeletConfig struct { // +optional CPUCfsQuota *bool `json:"cpuCfsQuota,omitempty"` // CPUCfsQuotaPeriod - Sets CPU CFS quota period value. + // Must end in "ms", e.g. "100ms" // +optional CPUCfsQuotaPeriod *string `json:"cpuCfsQuotaPeriod,omitempty"` // ImageGcHighThreshold - The percent of disk usage after which image garbage collection is always run. + // Valid values are 0-100 (inclusive). // +kubebuilder:validation:Minimum=0 // +kubebuilder:validation:Maximum=100 // +optional ImageGcHighThreshold *int32 `json:"imageGcHighThreshold,omitempty"` // ImageGcLowThreshold - The percent of disk usage before which image garbage collection is never run. + // Valid values are 0-100 (inclusive) and must be less than `imageGcHighThreshold`. // +kubebuilder:validation:Minimum=0 // +kubebuilder:validation:Maximum=100 // +optional @@ -120,12 +127,13 @@ type KubeletConfig struct { // +optional TopologyManagerPolicy *TopologyManagerPolicy `json:"topologyManagerPolicy,omitempty"` // AllowedUnsafeSysctls - Allowlist of unsafe sysctls or unsafe sysctl patterns (ending in `*`). + // Valid values match `kernel.shm*`, `kernel.msg*`, `kernel.sem`, `fs.mqueue.*`, or `net.*`. // +optional AllowedUnsafeSysctls []string `json:"allowedUnsafeSysctls,omitempty"` // FailSwapOn - If set to true it will make the Kubelet fail to start if swap is enabled on the node. // +optional FailSwapOn *bool `json:"failSwapOn,omitempty"` - // ContainerLogMaxSizeMB - The maximum size (e.g. 10Mi) of container log file before it is rotated. + // ContainerLogMaxSizeMB - The maximum size in MB of a container log file before it is rotated. // +optional ContainerLogMaxSizeMB *int32 `json:"containerLogMaxSizeMB,omitempty"` // ContainerLogMaxFiles - The maximum number of container log files that can be present for a container. The number must be ≥ 2. @@ -133,6 +141,7 @@ type KubeletConfig struct { // +optional ContainerLogMaxFiles *int32 `json:"containerLogMaxFiles,omitempty"` // PodMaxPids - The maximum number of processes per pod. + // Must not exceed kernel PID limit. -1 disables the limit. // +kubebuilder:validation:Minimum=-1 // +optional PodMaxPids *int32 `json:"podMaxPids,omitempty"` @@ -141,6 +150,7 @@ type KubeletConfig struct { // SysctlConfig specifies the settings for Linux agent nodes. type SysctlConfig struct { // FsAioMaxNr specifies the maximum number of system-wide asynchronous io requests. + // Valid values are 65536-6553500 (inclusive). // Maps to fs.aio-max-nr. // +kubebuilder:validation:Minimum=65536 // +kubebuilder:validation:Maximum=6553500 @@ -148,6 +158,7 @@ type SysctlConfig struct { FsAioMaxNr *int32 `json:"fsAioMaxNr,omitempty"` // FsFileMax specifies the max number of file-handles that the Linux kernel will allocate, by increasing increases the maximum number of open files permitted. + // Valid values are 8192-12000500 (inclusive). // Maps to fs.file-max. // +kubebuilder:validation:Minimum=8192 // +kubebuilder:validation:Maximum=12000500 @@ -155,6 +166,7 @@ type SysctlConfig struct { FsFileMax *int32 `json:"fsFileMax,omitempty"` // FsInotifyMaxUserWatches specifies the number of file watches allowed by the system. Each watch is roughly 90 bytes on a 32-bit kernel, and roughly 160 bytes on a 64-bit kernel. + // Valid values are 781250-2097152 (inclusive). // Maps to fs.inotify.max_user_watches. // +kubebuilder:validation:Minimum=781250 // +kubebuilder:validation:Maximum=2097152 @@ -162,6 +174,7 @@ type SysctlConfig struct { FsInotifyMaxUserWatches *int32 `json:"fsInotifyMaxUserWatches,omitempty"` // FsNrOpen specifies the maximum number of file-handles a process can allocate. + // Valid values are 8192-20000500 (inclusive). // Maps to fs.nr_open. // +kubebuilder:validation:Minimum=8192 // +kubebuilder:validation:Maximum=20000500 @@ -169,6 +182,7 @@ type SysctlConfig struct { FsNrOpen *int32 `json:"fsNrOpen,omitempty"` // KernelThreadsMax specifies the maximum number of all threads that can be created. + // Valid values are 20-513785 (inclusive). // Maps to kernel.threads-max. // +kubebuilder:validation:Minimum=20 // +kubebuilder:validation:Maximum=513785 @@ -176,6 +190,7 @@ type SysctlConfig struct { KernelThreadsMax *int32 `json:"kernelThreadsMax,omitempty"` // NetCoreNetdevMaxBacklog specifies maximum number of packets, queued on the INPUT side, when the interface receives packets faster than kernel can process them. + // Valid values are 1000-3240000 (inclusive). // Maps to net.core.netdev_max_backlog. // +kubebuilder:validation:Minimum=1000 // +kubebuilder:validation:Maximum=3240000 @@ -184,6 +199,7 @@ type SysctlConfig struct { // NetCoreOptmemMax specifies the maximum ancillary buffer size (option memory buffer) allowed per socket. // Socket option memory is used in a few cases to store extra structures relating to usage of the socket. + // Valid values are 20480-4194304 (inclusive). // Maps to net.core.optmem_max. // +kubebuilder:validation:Minimum=20480 // +kubebuilder:validation:Maximum=4194304 @@ -191,6 +207,7 @@ type SysctlConfig struct { NetCoreOptmemMax *int32 `json:"netCoreOptmemMax,omitempty"` // NetCoreRmemDefault specifies the default receive socket buffer size in bytes. + // Valid values are 212992-134217728 (inclusive). // Maps to net.core.rmem_default. // +kubebuilder:validation:Minimum=212992 // +kubebuilder:validation:Maximum=134217728 @@ -198,6 +215,7 @@ type SysctlConfig struct { NetCoreRmemDefault *int32 `json:"netCoreRmemDefault,omitempty"` // NetCoreRmemMax specifies the maximum receive socket buffer size in bytes. + // Valid values are 212992-134217728 (inclusive). // Maps to net.core.rmem_max. // +kubebuilder:validation:Minimum=212992 // +kubebuilder:validation:Maximum=134217728 @@ -207,6 +225,7 @@ type SysctlConfig struct { // NetCoreSomaxconn specifies maximum number of connection requests that can be queued for any given listening socket. // An upper limit for the value of the backlog parameter passed to the listen(2)(https://man7.org/linux/man-pages/man2/listen.2.html) function. // If the backlog argument is greater than the somaxconn, then it's silently truncated to this limit. + // Valid values are 4096-3240000 (inclusive). // Maps to net.core.somaxconn. // +kubebuilder:validation:Minimum=4096 // +kubebuilder:validation:Maximum=3240000 @@ -214,6 +233,7 @@ type SysctlConfig struct { NetCoreSomaxconn *int32 `json:"netCoreSomaxconn,omitempty"` // NetCoreWmemDefault specifies the default send socket buffer size in bytes. + // Valid values are 212992-134217728 (inclusive). // Maps to net.core.wmem_default. // +kubebuilder:validation:Minimum=212992 // +kubebuilder:validation:Maximum=134217728 @@ -221,6 +241,7 @@ type SysctlConfig struct { NetCoreWmemDefault *int32 `json:"netCoreWmemDefault,omitempty"` // NetCoreWmemMax specifies the maximum send socket buffer size in bytes. + // Valid values are 212992-134217728 (inclusive). // Maps to net.core.wmem_max. // +kubebuilder:validation:Minimum=212992 // +kubebuilder:validation:Maximum=134217728 @@ -237,6 +258,7 @@ type SysctlConfig struct { // NetIpv4NeighDefaultGcThresh1 specifies the minimum number of entries that may be in the ARP cache. // Garbage collection won't be triggered if the number of entries is below this setting. + // Valid values are 128-80000 (inclusive). // Maps to net.ipv4.neigh.default.gc_thresh1. // +kubebuilder:validation:Minimum=128 // +kubebuilder:validation:Maximum=80000 @@ -245,6 +267,7 @@ type SysctlConfig struct { // NetIpv4NeighDefaultGcThresh2 specifies soft maximum number of entries that may be in the ARP cache. // ARP garbage collection will be triggered about 5 seconds after reaching this soft maximum. + // Valid values are 512-90000 (inclusive). // Maps to net.ipv4.neigh.default.gc_thresh2. // +kubebuilder:validation:Minimum=512 // +kubebuilder:validation:Maximum=90000 @@ -252,6 +275,7 @@ type SysctlConfig struct { NetIpv4NeighDefaultGcThresh2 *int32 `json:"netIpv4NeighDefaultGcThresh2,omitempty"` // NetIpv4NeighDefaultGcThresh3 specified hard maximum number of entries in the ARP cache. + // Valid values are 1024-100000 (inclusive). // Maps to net.ipv4.neigh.default.gc_thresh3. // +kubebuilder:validation:Minimum=1024 // +kubebuilder:validation:Maximum=100000 @@ -259,6 +283,7 @@ type SysctlConfig struct { NetIpv4NeighDefaultGcThresh3 *int32 `json:"netIpv4NeighDefaultGcThresh3,omitempty"` // NetIpv4TCPFinTimeout specifies the length of time an orphaned connection will remain in the FIN_WAIT_2 state before it's aborted at the local end. + // Valid values are 5-120 (inclusive). // Maps to net.ipv4.tcp_fin_timeout. // +kubebuilder:validation:Minimum=5 // +kubebuilder:validation:Maximum=120 @@ -266,6 +291,7 @@ type SysctlConfig struct { NetIpv4TCPFinTimeout *int32 `json:"netIpv4TCPFinTimeout,omitempty"` // NetIpv4TCPKeepaliveProbes specifies the number of keepalive probes TCP sends out, until it decides the connection is broken. + // Valid values are 1-15 (inclusive). // Maps to net.ipv4.tcp_keepalive_probes. // +kubebuilder:validation:Minimum=1 // +kubebuilder:validation:Maximum=15 @@ -273,6 +299,7 @@ type SysctlConfig struct { NetIpv4TCPKeepaliveProbes *int32 `json:"netIpv4TCPKeepaliveProbes,omitempty"` // NetIpv4TCPKeepaliveTime specifies the rate at which TCP sends out a keepalive message when keepalive is enabled. + // Valid values are 30-432000 (inclusive). // Maps to net.ipv4.tcp_keepalive_time. // +kubebuilder:validation:Minimum=30 // +kubebuilder:validation:Maximum=432000 @@ -281,6 +308,7 @@ type SysctlConfig struct { // NetIpv4TCPMaxSynBacklog specifies the maximum number of queued connection requests that have still not received an acknowledgment from the connecting client. // If this number is exceeded, the kernel will begin dropping requests. + // Valid values are 128-3240000 (inclusive). // Maps to net.ipv4.tcp_max_syn_backlog. // +kubebuilder:validation:Minimum=128 // +kubebuilder:validation:Maximum=3240000 @@ -289,6 +317,7 @@ type SysctlConfig struct { // NetIpv4TCPMaxTwBuckets specifies maximal number of timewait sockets held by system simultaneously. // If this number is exceeded, time-wait socket is immediately destroyed and warning is printed. + // Valid values are 8000-1440000 (inclusive). // Maps to net.ipv4.tcp_max_tw_buckets. // +kubebuilder:validation:Minimum=8000 // +kubebuilder:validation:Maximum=1440000 @@ -302,6 +331,7 @@ type SysctlConfig struct { // NetIpv4TCPkeepaliveIntvl specifies the frequency of the probes sent out. // Multiplied by tcpKeepaliveprobes, it makes up the time to kill a connection that isn't responding, after probes started. + // Valid values are 1-75 (inclusive). // Maps to net.ipv4.tcp_keepalive_intvl. // +kubebuilder:validation:Minimum=1 // +kubebuilder:validation:Maximum=75 @@ -309,6 +339,7 @@ type SysctlConfig struct { NetIpv4TCPkeepaliveIntvl *int32 `json:"netIpv4TCPkeepaliveIntvl,omitempty"` // NetNetfilterNfConntrackBuckets specifies the size of hash table used by nf_conntrack module to record the established connection record of the TCP protocol. + // Valid values are 65536-147456 (inclusive). // Maps to net.netfilter.nf_conntrack_buckets. // +kubebuilder:validation:Minimum=65536 // +kubebuilder:validation:Maximum=147456 @@ -316,6 +347,7 @@ type SysctlConfig struct { NetNetfilterNfConntrackBuckets *int32 `json:"netNetfilterNfConntrackBuckets,omitempty"` // NetNetfilterNfConntrackMax specifies the maximum number of connections supported by the nf_conntrack module or the size of connection tracking table. + // Valid values are 131072-1048576 (inclusive). // Maps to net.netfilter.nf_conntrack_max. // +kubebuilder:validation:Minimum=131072 // +kubebuilder:validation:Maximum=1048576 @@ -324,6 +356,7 @@ type SysctlConfig struct { // VMMaxMapCount specifies the maximum number of memory map areas a process may have. // Maps to vm.max_map_count. + // Valid values are 65530-262144 (inclusive). // +kubebuilder:validation:Minimum=65530 // +kubebuilder:validation:Maximum=262144 // +optional @@ -331,6 +364,7 @@ type SysctlConfig struct { // VMSwappiness specifies aggressiveness of the kernel in swapping memory pages. // Higher values will increase aggressiveness, lower values decrease the amount of swap. + // Valid values are 0-100 (inclusive). // Maps to vm.swappiness. // +kubebuilder:validation:Minimum=0 // +kubebuilder:validation:Maximum=100 @@ -338,6 +372,7 @@ type SysctlConfig struct { VMSwappiness *int32 `json:"vmSwappiness,omitempty"` // VMVfsCachePressure specifies the percentage value that controls tendency of the kernel to reclaim the memory, which is used for caching of directory and inode objects. + // Valid values are 1-500 (inclusive). // Maps to vm.vfs_cache_pressure. // +kubebuilder:validation:Minimum=1 // +kubebuilder:validation:Maximum=500 @@ -346,9 +381,16 @@ type SysctlConfig struct { } // LinuxOSConfig specifies the custom Linux OS settings and configurations. +// See also [AKS doc]. +// +// [AKS doc]: https://learn.microsoft.com/azure/aks/custom-node-configuration#linux-os-custom-configuration type LinuxOSConfig struct { // SwapFileSizeMB specifies size in MB of a swap file will be created on the agent nodes from this node pool. - // Max value of SwapFileSizeMB should be the size of temporary disk(/dev/sdb). Refer: https://learn.microsoft.com/en-us/azure/virtual-machines/managed-disks-overview#temporary-disk + // Max value of SwapFileSizeMB should be the size of temporary disk(/dev/sdb). + // Must be at least 1. + // See also [AKS doc]. + // + // [AKS doc]: https://learn.microsoft.com/azure/virtual-machines/managed-disks-overview#temporary-disk // +kubebuilder:validation:Minimum=1 // +optional SwapFileSizeMB *int32 `json:"swapFileSizeMB,omitempty"` @@ -358,12 +400,17 @@ type LinuxOSConfig struct { Sysctls *SysctlConfig `json:"sysctls,omitempty"` // TransparentHugePageDefrag specifies whether the kernel should make aggressive use of memory compaction to make more hugepages available. - // Refer to https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html#admin-guide-transhuge for more details. + // See also [Linux doc]. + // + // [Linux doc]: https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html#admin-guide-transhuge for more details. // +kubebuilder:validation:Enum=always;defer;defer+madvise;madvise;never // +optional TransparentHugePageDefrag *TransparentHugePageOption `json:"transparentHugePageDefrag,omitempty"` - // TransparentHugePageEnabled specifies various modes of Transparent Hugepages. Refer to https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html#admin-guide-transhuge for more details. + // TransparentHugePageEnabled specifies various modes of Transparent Hugepages. + // See also [Linux doc]. + // + // [Linux doc]: https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html#admin-guide-transhuge for more details. // +kubebuilder:validation:Enum=always;madvise;never // +optional TransparentHugePageEnabled *TransparentHugePageOption `json:"transparentHugePageEnabled,omitempty"` @@ -378,6 +425,7 @@ type AzureManagedMachinePoolSpec struct { AdditionalTags Tags `json:"additionalTags,omitempty"` // Name - name of the agent pool. If not specified, CAPZ uses the name of the CR as the agent pool name. + // Immutable. // +optional Name *string `json:"name,omitempty"` @@ -386,22 +434,31 @@ type AzureManagedMachinePoolSpec struct { Mode string `json:"mode"` // SKU is the size of the VMs in the node pool. + // Immutable. SKU string `json:"sku"` // OSDiskSizeGB is the disk size for every machine in this agent pool. // If you specify 0, it will apply the default osDisk size according to the vmSize specified. + // Immutable. // +optional OSDiskSizeGB *int32 `json:"osDiskSizeGB,omitempty"` // AvailabilityZones - Availability zones for nodes. Must use VirtualMachineScaleSets AgentPoolType. + // Immutable. // +optional AvailabilityZones []string `json:"availabilityZones,omitempty"` - // Node labels - labels for all of the nodes present in node pool + // Node labels - labels for all of the nodes present in node pool. + // See also [AKS doc]. + // + // [AKS doc]: https://learn.microsoft.com/azure/aks/use-labels // +optional NodeLabels map[string]string `json:"nodeLabels,omitempty"` // Taints specifies the taints for nodes present in this agent pool. + // See also [AKS doc]. + // + // [AKS doc]: https://learn.microsoft.com/azure/aks/use-multiple-node-pools#setting-node-pool-taints // +optional Taints Taints `json:"taints,omitempty"` @@ -413,59 +470,86 @@ type AzureManagedMachinePoolSpec struct { // +optional Scaling *ManagedMachinePoolScaling `json:"scaling,omitempty"` - // MaxPods specifies the kubelet --max-pods configuration for the node pool. + // MaxPods specifies the kubelet `--max-pods` configuration for the node pool. + // Immutable. + // See also [AKS doc], [K8s doc]. + // + // [AKS doc]: https://learn.microsoft.com/azure/aks/configure-azure-cni#configure-maximum---new-clusters + // [K8s doc]: https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/ // +optional MaxPods *int32 `json:"maxPods,omitempty"` - // OsDiskType specifies the OS disk type for each node in the pool. Allowed values are 'Ephemeral' and 'Managed'. + // OsDiskType specifies the OS disk type for each node in the pool. Allowed values are 'Ephemeral' and 'Managed' (default). + // Immutable. + // See also [AKS doc]. + // + // [AKS doc]: https://learn.microsoft.com/azure/aks/cluster-configuration#ephemeral-os // +kubebuilder:validation:Enum=Ephemeral;Managed // +kubebuilder:default=Managed // +optional OsDiskType *string `json:"osDiskType,omitempty"` // EnableUltraSSD enables the storage type UltraSSD_LRS for the agent pool. + // Immutable. // +optional EnableUltraSSD *bool `json:"enableUltraSSD,omitempty"` - // OSType specifies the virtual machine operating system. Default to Linux. Possible values include: 'Linux', 'Windows' + // OSType specifies the virtual machine operating system. Default to Linux. Possible values include: 'Linux', 'Windows'. + // 'Windows' requires the AzureManagedControlPlane's `spec.networkPlugin` to be `azure`. + // Immutable. + // See also [AKS doc]. + // + // [AKS doc]: https://learn.microsoft.com/rest/api/aks/agent-pools/create-or-update?tabs=HTTP#ostype // +kubebuilder:validation:Enum=Linux;Windows // +optional OSType *string `json:"osType,omitempty"` // EnableNodePublicIP controls whether or not nodes in the pool each have a public IP address. + // Immutable. // +optional EnableNodePublicIP *bool `json:"enableNodePublicIP,omitempty"` // NodePublicIPPrefixID specifies the public IP prefix resource ID which VM nodes should use IPs from. + // Immutable. // +optional NodePublicIPPrefixID *string `json:"nodePublicIPPrefixID,omitempty"` // ScaleSetPriority specifies the ScaleSetPriority value. Default to Regular. Possible values include: 'Regular', 'Spot' + // Immutable. // +kubebuilder:validation:Enum=Regular;Spot // +optional ScaleSetPriority *string `json:"scaleSetPriority,omitempty"` // KubeletConfig specifies the kubelet configurations for nodes. + // Immutable. // +optional KubeletConfig *KubeletConfig `json:"kubeletConfig,omitempty"` // KubeletDiskType specifies the kubelet disk type. Default to OS. Possible values include: 'OS', 'Temporary'. - // Requires kubeletDisk preview feature to be set. + // Requires Microsoft.ContainerService/KubeletDisk preview feature to be set. + // Immutable. + // See also [AKS doc]. + // + // [AKS doc]: https://learn.microsoft.com/rest/api/aks/agent-pools/create-or-update?tabs=HTTP#kubeletdisktype // +kubebuilder:validation:Enum=OS;Temporary // +optional KubeletDiskType *KubeletDiskType `json:"kubeletDiskType,omitempty"` // LinuxOSConfig specifies the custom Linux OS settings and configurations. + // Immutable. // +optional LinuxOSConfig *LinuxOSConfig `json:"linuxOSConfig,omitempty"` // SubnetName specifies the Subnet where the MachinePool will be placed + // Immutable. // +optional SubnetName *string `json:"subnetName,omitempty"` } // ManagedMachinePoolScaling specifies scaling options. type ManagedMachinePoolScaling struct { + // MinSize is the minimum number of nodes for auto-scaling. MinSize *int32 `json:"minSize,omitempty"` + // MaxSize is the maximum number of nodes for auto-scaling. MaxSize *int32 `json:"maxSize,omitempty"` } diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedclusters.yaml index 19c67f74c1c..dc426ec5803 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedclusters.yaml @@ -42,7 +42,8 @@ spec: properties: controlPlaneEndpoint: description: ControlPlaneEndpoint represents the endpoint used to - communicate with the control plane. + communicate with the control plane. Immutable, populated by the + AKS API at create. properties: host: description: The hostname on which the API server is serving. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml index 551dd56471e..c26d17abbf9 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml @@ -88,7 +88,7 @@ spec: type: array apiServerAccessProfile: description: APIServerAccessProfile is the access profile for AKS - API server. + API server. Immutable except for `authorizedIPRanges`. properties: authorizedIPRanges: description: AuthorizedIPRanges - Authorized IP Ranges to kubernetes @@ -225,7 +225,8 @@ spec: type: string controlPlaneEndpoint: description: ControlPlaneEndpoint represents the endpoint used to - communicate with the control plane. + communicate with the control plane. Immutable, populated by the + AKS API at create. properties: host: description: The hostname on which the API server is serving. @@ -241,7 +242,7 @@ spec: dnsServiceIP: description: DNSServiceIP is an IP address assigned to the Kubernetes DNS service. It must be within the Kubernetes service address range - specified in serviceCidr. + specified in serviceCidr. Immutable. type: string identityRef: description: IdentityRef is a reference to a AzureClusterIdentity @@ -318,23 +319,25 @@ spec: type: object loadBalancerSKU: description: LoadBalancerSKU is the SKU of the loadBalancer to be - provisioned. + provisioned. Immutable. enum: - Basic - Standard type: string location: description: 'Location is a string matching one of the canonical Azure - region names. Examples: "westus2", "eastus".' + region names. Examples: "westus2", "eastus". Immutable.' type: string networkPlugin: - description: NetworkPlugin used for building Kubernetes network. + description: NetworkPlugin used for building Kubernetes network. Allowed + values are "azure", "kubenet". Immutable. enum: - azure - kubenet type: string networkPolicy: - description: NetworkPolicy used for building Kubernetes network. + description: NetworkPolicy used for building Kubernetes network. Allowed + values are "azure", "calico". Immutable. enum: - azure - calico @@ -342,10 +345,10 @@ spec: nodeResourceGroupName: description: NodeResourceGroupName is the name of the resource group containing cluster IaaS resources. Will be populated to default - in webhook. + in webhook. Immutable. type: string outboundType: - description: Outbound configuration used by Nodes. + description: Outbound configuration used by Nodes. Immutable. enum: - loadBalancer - managedNATGateway @@ -354,7 +357,7 @@ spec: type: string resourceGroupName: description: ResourceGroupName is the name of the Azure resource group - for this AKS Cluster. + for this AKS Cluster. Immutable. type: string sku: description: SKU is the SKU of the AKS to be provisioned. @@ -370,11 +373,11 @@ spec: type: object sshPublicKey: description: SSHPublicKey is a string literal containing an ssh public - key base64 encoded. + key base64 encoded. Immutable. type: string subscriptionID: description: SubscriptionID is the GUID of the Azure subscription - to hold this cluster. + to hold this cluster. Immutable. type: string version: description: Version defines the desired Kubernetes version. @@ -382,7 +385,7 @@ spec: type: string virtualNetwork: description: VirtualNetwork describes the vnet for the AKS cluster. - Will be created if it does not exist. + Will be created if it does not exist. Immutable except for `subnet`. properties: cidrBlock: type: string @@ -393,8 +396,7 @@ spec: for the VNet and Subnet. type: string subnet: - description: ManagedControlPlaneSubnet describes a subnet for - an AKS cluster. + description: Immutable except for `serviceEndpoints`. properties: cidrBlock: type: string diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml index c92ee8d68e4..cac7edf571c 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml @@ -54,25 +54,27 @@ spec: type: object availabilityZones: description: AvailabilityZones - Availability zones for nodes. Must - use VirtualMachineScaleSets AgentPoolType. + use VirtualMachineScaleSets AgentPoolType. Immutable. items: type: string type: array enableNodePublicIP: description: EnableNodePublicIP controls whether or not nodes in the - pool each have a public IP address. + pool each have a public IP address. Immutable. type: boolean enableUltraSSD: description: EnableUltraSSD enables the storage type UltraSSD_LRS - for the agent pool. + for the agent pool. Immutable. type: boolean kubeletConfig: description: KubeletConfig specifies the kubelet configurations for - nodes. + nodes. Immutable. properties: allowedUnsafeSysctls: description: AllowedUnsafeSysctls - Allowlist of unsafe sysctls - or unsafe sysctl patterns (ending in `*`). + or unsafe sysctl patterns (ending in `*`). Valid values match + `kernel.shm*`, `kernel.msg*`, `kernel.sem`, `fs.mqueue.*`, or + `net.*`. items: type: string type: array @@ -84,8 +86,8 @@ spec: minimum: 2 type: integer containerLogMaxSizeMB: - description: ContainerLogMaxSizeMB - The maximum size (e.g. 10Mi) - of container log file before it is rotated. + description: ContainerLogMaxSizeMB - The maximum size in MB of + a container log file before it is rotated. format: int32 type: integer cpuCfsQuota: @@ -94,6 +96,7 @@ spec: type: boolean cpuCfsQuotaPeriod: description: CPUCfsQuotaPeriod - Sets CPU CFS quota period value. + Must end in "ms", e.g. "100ms" type: string cpuManagerPolicy: description: CPUManagerPolicy - CPU Manager policy to use. @@ -107,21 +110,23 @@ spec: type: boolean imageGcHighThreshold: description: ImageGcHighThreshold - The percent of disk usage - after which image garbage collection is always run. + after which image garbage collection is always run. Valid values + are 0-100 (inclusive). format: int32 maximum: 100 minimum: 0 type: integer imageGcLowThreshold: description: ImageGcLowThreshold - The percent of disk usage before - which image garbage collection is never run. + which image garbage collection is never run. Valid values are + 0-100 (inclusive) and must be less than `imageGcHighThreshold`. format: int32 maximum: 100 minimum: 0 type: integer podMaxPids: description: PodMaxPids - The maximum number of processes per - pod. + pod. Must not exceed kernel PID limit. -1 disables the limit. format: int32 minimum: -1 type: integer @@ -136,22 +141,23 @@ spec: type: string type: object kubeletDiskType: - description: 'KubeletDiskType specifies the kubelet disk type. Default - to OS. Possible values include: ''OS'', ''Temporary''. Requires - kubeletDisk preview feature to be set.' + description: "KubeletDiskType specifies the kubelet disk type. Default + to OS. Possible values include: 'OS', 'Temporary'. Requires Microsoft.ContainerService/KubeletDisk + preview feature to be set. Immutable. See also [AKS doc]. \n [AKS + doc]: https://learn.microsoft.com/rest/api/aks/agent-pools/create-or-update?tabs=HTTP#kubeletdisktype" enum: - OS - Temporary type: string linuxOSConfig: description: LinuxOSConfig specifies the custom Linux OS settings - and configurations. + and configurations. Immutable. properties: swapFileSizeMB: - description: 'SwapFileSizeMB specifies size in MB of a swap file + description: "SwapFileSizeMB specifies size in MB of a swap file will be created on the agent nodes from this node pool. Max value of SwapFileSizeMB should be the size of temporary disk(/dev/sdb). - Refer: https://learn.microsoft.com/en-us/azure/virtual-machines/managed-disks-overview#temporary-disk' + Must be at least 1. See also [AKS doc]. \n [AKS doc]: https://learn.microsoft.com/azure/virtual-machines/managed-disks-overview#temporary-disk" format: int32 minimum: 1 type: integer @@ -160,7 +166,8 @@ spec: properties: fsAioMaxNr: description: FsAioMaxNr specifies the maximum number of system-wide - asynchronous io requests. Maps to fs.aio-max-nr. + asynchronous io requests. Valid values are 65536-6553500 + (inclusive). Maps to fs.aio-max-nr. format: int32 maximum: 6553500 minimum: 65536 @@ -168,7 +175,8 @@ spec: fsFileMax: description: FsFileMax specifies the max number of file-handles that the Linux kernel will allocate, by increasing increases - the maximum number of open files permitted. Maps to fs.file-max. + the maximum number of open files permitted. Valid values + are 8192-12000500 (inclusive). Maps to fs.file-max. format: int32 maximum: 12000500 minimum: 8192 @@ -177,21 +185,24 @@ spec: description: FsInotifyMaxUserWatches specifies the number of file watches allowed by the system. Each watch is roughly 90 bytes on a 32-bit kernel, and roughly 160 bytes on a - 64-bit kernel. Maps to fs.inotify.max_user_watches. + 64-bit kernel. Valid values are 781250-2097152 (inclusive). + Maps to fs.inotify.max_user_watches. format: int32 maximum: 2097152 minimum: 781250 type: integer fsNrOpen: description: FsNrOpen specifies the maximum number of file-handles - a process can allocate. Maps to fs.nr_open. + a process can allocate. Valid values are 8192-20000500 (inclusive). + Maps to fs.nr_open. format: int32 maximum: 20000500 minimum: 8192 type: integer kernelThreadsMax: description: KernelThreadsMax specifies the maximum number - of all threads that can be created. Maps to kernel.threads-max. + of all threads that can be created. Valid values are 20-513785 + (inclusive). Maps to kernel.threads-max. format: int32 maximum: 513785 minimum: 20 @@ -199,8 +210,8 @@ spec: netCoreNetdevMaxBacklog: description: NetCoreNetdevMaxBacklog specifies maximum number of packets, queued on the INPUT side, when the interface - receives packets faster than kernel can process them. Maps - to net.core.netdev_max_backlog. + receives packets faster than kernel can process them. Valid + values are 1000-3240000 (inclusive). Maps to net.core.netdev_max_backlog. format: int32 maximum: 3240000 minimum: 1000 @@ -209,21 +220,24 @@ spec: description: NetCoreOptmemMax specifies the maximum ancillary buffer size (option memory buffer) allowed per socket. Socket option memory is used in a few cases to store extra structures - relating to usage of the socket. Maps to net.core.optmem_max. + relating to usage of the socket. Valid values are 20480-4194304 + (inclusive). Maps to net.core.optmem_max. format: int32 maximum: 4194304 minimum: 20480 type: integer netCoreRmemDefault: description: NetCoreRmemDefault specifies the default receive - socket buffer size in bytes. Maps to net.core.rmem_default. + socket buffer size in bytes. Valid values are 212992-134217728 + (inclusive). Maps to net.core.rmem_default. format: int32 maximum: 134217728 minimum: 212992 type: integer netCoreRmemMax: description: NetCoreRmemMax specifies the maximum receive - socket buffer size in bytes. Maps to net.core.rmem_max. + socket buffer size in bytes. Valid values are 212992-134217728 + (inclusive). Maps to net.core.rmem_max. format: int32 maximum: 134217728 minimum: 212992 @@ -234,21 +248,24 @@ spec: socket. An upper limit for the value of the backlog parameter passed to the listen(2)(https://man7.org/linux/man-pages/man2/listen.2.html) function. If the backlog argument is greater than the somaxconn, - then it's silently truncated to this limit. Maps to net.core.somaxconn. + then it's silently truncated to this limit. Valid values + are 4096-3240000 (inclusive). Maps to net.core.somaxconn. format: int32 maximum: 3240000 minimum: 4096 type: integer netCoreWmemDefault: description: NetCoreWmemDefault specifies the default send - socket buffer size in bytes. Maps to net.core.wmem_default. + socket buffer size in bytes. Valid values are 212992-134217728 + (inclusive). Maps to net.core.wmem_default. format: int32 maximum: 134217728 minimum: 212992 type: integer netCoreWmemMax: description: NetCoreWmemMax specifies the maximum send socket - buffer size in bytes. Maps to net.core.wmem_max. + buffer size in bytes. Valid values are 212992-134217728 + (inclusive). Maps to net.core.wmem_max. format: int32 maximum: 134217728 minimum: 212992 @@ -264,7 +281,8 @@ spec: description: NetIpv4NeighDefaultGcThresh1 specifies the minimum number of entries that may be in the ARP cache. Garbage collection won't be triggered if the number of entries is - below this setting. Maps to net.ipv4.neigh.default.gc_thresh1. + below this setting. Valid values are 128-80000 (inclusive). + Maps to net.ipv4.neigh.default.gc_thresh1. format: int32 maximum: 80000 minimum: 128 @@ -273,14 +291,16 @@ spec: description: NetIpv4NeighDefaultGcThresh2 specifies soft maximum number of entries that may be in the ARP cache. ARP garbage collection will be triggered about 5 seconds after reaching - this soft maximum. Maps to net.ipv4.neigh.default.gc_thresh2. + this soft maximum. Valid values are 512-90000 (inclusive). + Maps to net.ipv4.neigh.default.gc_thresh2. format: int32 maximum: 90000 minimum: 512 type: integer netIpv4NeighDefaultGcThresh3: description: NetIpv4NeighDefaultGcThresh3 specified hard maximum - number of entries in the ARP cache. Maps to net.ipv4.neigh.default.gc_thresh3. + number of entries in the ARP cache. Valid values are 1024-100000 + (inclusive). Maps to net.ipv4.neigh.default.gc_thresh3. format: int32 maximum: 100000 minimum: 1024 @@ -288,7 +308,8 @@ spec: netIpv4TCPFinTimeout: description: NetIpv4TCPFinTimeout specifies the length of time an orphaned connection will remain in the FIN_WAIT_2 - state before it's aborted at the local end. Maps to net.ipv4.tcp_fin_timeout. + state before it's aborted at the local end. Valid values + are 5-120 (inclusive). Maps to net.ipv4.tcp_fin_timeout. format: int32 maximum: 120 minimum: 5 @@ -296,7 +317,8 @@ spec: netIpv4TCPKeepaliveProbes: description: NetIpv4TCPKeepaliveProbes specifies the number of keepalive probes TCP sends out, until it decides the - connection is broken. Maps to net.ipv4.tcp_keepalive_probes. + connection is broken. Valid values are 1-15 (inclusive). + Maps to net.ipv4.tcp_keepalive_probes. format: int32 maximum: 15 minimum: 1 @@ -304,7 +326,8 @@ spec: netIpv4TCPKeepaliveTime: description: NetIpv4TCPKeepaliveTime specifies the rate at which TCP sends out a keepalive message when keepalive is - enabled. Maps to net.ipv4.tcp_keepalive_time. + enabled. Valid values are 30-432000 (inclusive). Maps to + net.ipv4.tcp_keepalive_time. format: int32 maximum: 432000 minimum: 30 @@ -314,7 +337,8 @@ spec: number of queued connection requests that have still not received an acknowledgment from the connecting client. If this number is exceeded, the kernel will begin dropping - requests. Maps to net.ipv4.tcp_max_syn_backlog. + requests. Valid values are 128-3240000 (inclusive). Maps + to net.ipv4.tcp_max_syn_backlog. format: int32 maximum: 3240000 minimum: 128 @@ -323,7 +347,8 @@ spec: description: NetIpv4TCPMaxTwBuckets specifies maximal number of timewait sockets held by system simultaneously. If this number is exceeded, time-wait socket is immediately destroyed - and warning is printed. Maps to net.ipv4.tcp_max_tw_buckets. + and warning is printed. Valid values are 8000-1440000 (inclusive). + Maps to net.ipv4.tcp_max_tw_buckets. format: int32 maximum: 1440000 minimum: 8000 @@ -337,7 +362,8 @@ spec: description: NetIpv4TCPkeepaliveIntvl specifies the frequency of the probes sent out. Multiplied by tcpKeepaliveprobes, it makes up the time to kill a connection that isn't responding, - after probes started. Maps to net.ipv4.tcp_keepalive_intvl. + after probes started. Valid values are 1-75 (inclusive). + Maps to net.ipv4.tcp_keepalive_intvl. format: int32 maximum: 75 minimum: 1 @@ -345,8 +371,8 @@ spec: netNetfilterNfConntrackBuckets: description: NetNetfilterNfConntrackBuckets specifies the size of hash table used by nf_conntrack module to record - the established connection record of the TCP protocol. Maps - to net.netfilter.nf_conntrack_buckets. + the established connection record of the TCP protocol. Valid + values are 65536-147456 (inclusive). Maps to net.netfilter.nf_conntrack_buckets. format: int32 maximum: 147456 minimum: 65536 @@ -354,7 +380,8 @@ spec: netNetfilterNfConntrackMax: description: NetNetfilterNfConntrackMax specifies the maximum number of connections supported by the nf_conntrack module - or the size of connection tracking table. Maps to net.netfilter.nf_conntrack_max. + or the size of connection tracking table. Valid values are + 131072-1048576 (inclusive). Maps to net.netfilter.nf_conntrack_max. format: int32 maximum: 1048576 minimum: 131072 @@ -362,6 +389,7 @@ spec: vmMaxMapCount: description: VMMaxMapCount specifies the maximum number of memory map areas a process may have. Maps to vm.max_map_count. + Valid values are 65530-262144 (inclusive). format: int32 maximum: 262144 minimum: 65530 @@ -370,7 +398,7 @@ spec: description: VMSwappiness specifies aggressiveness of the kernel in swapping memory pages. Higher values will increase aggressiveness, lower values decrease the amount of swap. - Maps to vm.swappiness. + Valid values are 0-100 (inclusive). Maps to vm.swappiness. format: int32 maximum: 100 minimum: 0 @@ -379,17 +407,18 @@ spec: description: VMVfsCachePressure specifies the percentage value that controls tendency of the kernel to reclaim the memory, which is used for caching of directory and inode objects. - Maps to vm.vfs_cache_pressure. + Valid values are 1-500 (inclusive). Maps to vm.vfs_cache_pressure. format: int32 maximum: 500 minimum: 1 type: integer type: object transparentHugePageDefrag: - description: TransparentHugePageDefrag specifies whether the kernel - should make aggressive use of memory compaction to make more - hugepages available. Refer to https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html#admin-guide-transhuge - for more details. + description: "TransparentHugePageDefrag specifies whether the + kernel should make aggressive use of memory compaction to make + more hugepages available. See also [Linux doc]. \n [Linux doc]: + https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html#admin-guide-transhuge + for more details." enum: - always - defer @@ -398,9 +427,10 @@ spec: - never type: string transparentHugePageEnabled: - description: TransparentHugePageEnabled specifies various modes - of Transparent Hugepages. Refer to https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html#admin-guide-transhuge - for more details. + description: "TransparentHugePageEnabled specifies various modes + of Transparent Hugepages. See also [Linux doc]. \n [Linux doc]: + https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html#admin-guide-transhuge + for more details." enum: - always - madvise @@ -408,8 +438,10 @@ spec: type: string type: object maxPods: - description: MaxPods specifies the kubelet --max-pods configuration - for the node pool. + description: "MaxPods specifies the kubelet `--max-pods` configuration + for the node pool. Immutable. See also [AKS doc], [K8s doc]. \n + [AKS doc]: https://learn.microsoft.com/azure/aks/configure-azure-cni#configure-maximum---new-clusters + [K8s doc]: https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/" format: int32 type: integer mode: @@ -421,35 +453,38 @@ spec: type: string name: description: Name - name of the agent pool. If not specified, CAPZ - uses the name of the CR as the agent pool name. + uses the name of the CR as the agent pool name. Immutable. type: string nodeLabels: additionalProperties: type: string - description: Node labels - labels for all of the nodes present in - node pool + description: "Node labels - labels for all of the nodes present in + node pool. See also [AKS doc]. \n [AKS doc]: https://learn.microsoft.com/azure/aks/use-labels" type: object nodePublicIPPrefixID: description: NodePublicIPPrefixID specifies the public IP prefix resource - ID which VM nodes should use IPs from. + ID which VM nodes should use IPs from. Immutable. type: string osDiskSizeGB: description: OSDiskSizeGB is the disk size for every machine in this agent pool. If you specify 0, it will apply the default osDisk size - according to the vmSize specified. + according to the vmSize specified. Immutable. format: int32 type: integer osDiskType: default: Managed - description: OsDiskType specifies the OS disk type for each node in - the pool. Allowed values are 'Ephemeral' and 'Managed'. + description: "OsDiskType specifies the OS disk type for each node + in the pool. Allowed values are 'Ephemeral' and 'Managed' (default). + Immutable. See also [AKS doc]. \n [AKS doc]: https://learn.microsoft.com/azure/aks/cluster-configuration#ephemeral-os" enum: - Ephemeral - Managed type: string osType: - description: 'OSType specifies the virtual machine operating system. - Default to Linux. Possible values include: ''Linux'', ''Windows''' + description: "OSType specifies the virtual machine operating system. + Default to Linux. Possible values include: 'Linux', 'Windows'. 'Windows' + requires the AzureManagedControlPlane's `spec.networkPlugin` to + be `azure`. Immutable. See also [AKS doc]. \n [AKS doc]: https://learn.microsoft.com/rest/api/aks/agent-pools/create-or-update?tabs=HTTP#ostype" enum: - Linux - Windows @@ -462,7 +497,8 @@ spec: type: array scaleSetPriority: description: 'ScaleSetPriority specifies the ScaleSetPriority value. - Default to Regular. Possible values include: ''Regular'', ''Spot''' + Default to Regular. Possible values include: ''Regular'', ''Spot'' + Immutable.' enum: - Regular - Spot @@ -472,22 +508,24 @@ spec: node pool. properties: maxSize: + description: MaxSize is the maximum number of nodes for auto-scaling. format: int32 type: integer minSize: + description: MinSize is the minimum number of nodes for auto-scaling. format: int32 type: integer type: object sku: - description: SKU is the size of the VMs in the node pool. + description: SKU is the size of the VMs in the node pool. Immutable. type: string subnetName: description: SubnetName specifies the Subnet where the MachinePool - will be placed + will be placed Immutable. type: string taints: - description: Taints specifies the taints for nodes present in this - agent pool. + description: "Taints specifies the taints for nodes present in this + agent pool. See also [AKS doc]. \n [AKS doc]: https://learn.microsoft.com/azure/aks/use-multiple-node-pools#setting-node-pool-taints" items: description: Taint represents a Kubernetes taint. properties: diff --git a/docs/book/src/topics/managedcluster.md b/docs/book/src/topics/managedcluster.md index 499f077e70a..e015fadeee0 100644 --- a/docs/book/src/topics/managedcluster.md +++ b/docs/book/src/topics/managedcluster.md @@ -16,7 +16,7 @@ corresponds to provisioning an AKS cluster. AzureManagedMachinePool corresponds one-to-one with AKS node pools. This also means that creating an AzureManagedControlPlane requires at least one AzureManagedMachinePool with `spec.mode` `System`, since AKS expects at least one system pool at creation -time. For more documentation on system node pool refer [AKS Docs](https://docs.microsoft.com/en-us/azure/aks/use-system-pools) +time. For more documentation on system node pool refer [AKS Docs](https://learn.microsoft.com/azure/aks/use-system-pools) ## Deploy with clusterctl @@ -63,7 +63,7 @@ Managed clusters require the Cluster API "MachinePool" feature flag enabled. You export EXP_MACHINE_POOL=true ``` -Optionally, the you can enable the CAPZ "AKSResourceHealth" feature flag as well: +Optionally, you can enable the CAPZ "AKSResourceHealth" feature flag as well: ```bash export EXP_AKS_RESOURCE_HEALTH=true @@ -197,20 +197,14 @@ spec: Please note that we don't declare a configuration for the apiserver endpoint. This configuration data will be populated automatically based on the data returned from AKS API during cluster create as `.spec.controlPlaneEndpoint.Host` and `.spec.controlPlaneEndpoint.Port` in both the `AzureManagedCluster` and `AzureManagedControlPlane` resources. Any user-provided data will be ignored and overwritten by data returned from the AKS API. +The [CAPZ API reference documentation](../reference/v1beta1-api.html) describes all of the available options. See also the AKS API documentation for [Agent Pools](https://learn.microsoft.com/rest/api/aks/agent-pools/create-or-update?tabs=HTTP) and [Managed Clusters](https://learn.microsoft.com/rest/api/aks/managed-clusters/create-or-update?tabs=HTTP). + The main features for configuration are: -- [networkPolicy](https://docs.microsoft.com/en-us/azure/aks/concepts-network#network-policies) -- [networkPlugin](https://docs.microsoft.com/en-us/azure/aks/concepts-network#azure-virtual-networks) +- [networkPolicy](https://learn.microsoft.com/azure/aks/concepts-network#network-policies) +- [networkPlugin](https://learn.microsoft.com/azure/aks/concepts-network#azure-virtual-networks) - [addonProfiles](https://learn.microsoft.com/cli/azure/aks/addon?view=azure-cli-latest#az-aks-addon-list-available) - for additional addons not listed below, look for the `*ADDON_NAME` values in [this code](https://github.com/Azure/azure-cli/blob/main/src/azure-cli/azure/cli/command_modules/acs/_consts.py). -Other configuration values like subscriptionId and node machine type -should be fairly clear from context. - -| option | available values | -|---------------------------|-------------------------------| -| networkPlugin | azure, kubenet | -| networkPolicy | azure, calico | - | addon name | YAML value | |---------------------------|---------------------------| | http_application_routing | httpApplicationRouting | @@ -249,297 +243,10 @@ spec: name: test-subnet ``` -### Multitenancy - -Multitenancy for managed clusters can be configured by using `aks-multi-tenancy` flavor. The steps for creating an azure managed identity and mapping it to an `AzureClusterIdentity` are similar to the ones described [here](https://capz.sigs.k8s.io/topics/multitenancy.html). -The `AzureClusterIdentity` object is then mapped to a managed cluster through the `identityRef` field in `AzureManagedControlPlane.spec`. -Following is an example configuration: - -```yaml -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - services: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: exp.infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureManagedControlPlane - name: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: exp.infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureManagedCluster - name: ${CLUSTER_NAME} ---- -apiVersion: exp.infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureManagedControlPlane -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - identityRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureClusterIdentity - name: ${CLUSTER_IDENTITY_NAME} - namespace: ${CLUSTER_IDENTITY_NAMESPACE} - location: ${AZURE_LOCATION} - resourceGroupName: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} - version: ${KUBERNETES_VERSION} ---- -``` - -### AKS Managed Azure Active Directory Integration - -Azure Kubernetes Service can be configured to use Azure Active Directory for user authentication. -AAD for managed clusters can be configured by enabling the `managed` spec in `AzureManagedControlPlane` to `true` -and by providing Azure AD GroupObjectId in `AdminGroupObjectIDs` array. The group is needed as admin group for -the cluster to grant cluster admin permissions. You can use an existing Azure AD group, or create a new one. For more documentation about AAD refer [AKS AAD Docs](https://docs.microsoft.com/en-us/azure/aks/managed-aad) - -```yaml -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureManagedControlPlane -metadata: - name: my-cluster-control-plane -spec: - location: southcentralus - resourceGroupName: foo-bar - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - subscriptionID: fae7cc14-bfba-4471-9435-f945b42a16dd # fake uuid - version: v1.21.2 - aadProfile: - managed: true - adminGroupObjectIDs: - - 917056a9-8eb5-439c-g679-b34901ade75h # fake admin groupId -``` - -### AKS Cluster Autoscaler - -Azure Kubernetes Service can have the cluster autoscaler enabled by specifying `scaling` spec in any of the `AzureManagedMachinePool` defined. - -```yaml -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureManagedMachinePool -metadata: - name: agentpool0 -spec: - mode: System - osDiskSizeGB: 30 - sku: Standard_D2s_v3 - scaling: - minSize: 2 - maxSize: 10 -``` - -The cluster autoscaler behavior settings can be set in the `AzureManagedControlPlane`. Not setting a property will default to the value used by AKS. All values are expected to be strings. - -```yaml -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureManagedControlPlane -metadata: - name: my-cluster-control-plane -spec: - autoscalerProfile: - balanceSimilarNodeGroups: "false" - expander: "random" - maxEmptyBulkDelete: "10" - maxGracefulTerminationSec: "600" - maxNodeProvisionTime: "15m" - maxTotalUnreadyPercentage: "45" - newPodScaleUpDelay: "0s" - okTotalUnreadyCount: "3" - scanInterval: "10s" - scaleDownDelayAfterAdd: "10m" - scaleDownDelayAfterDelete: "10s" - scaleDownDelayAfterFailure: "3m" - scaleDownUnneededTime: "10m" - scaleDownUnreadyTime: "20m" - scaleDownUtilizationThreshold: "0.5" - skipNodesWithLocalStorage: "false" - skipNodesWithSystemPods: "true" -``` - -### AKS Node Labels to an Agent Pool - -You can configure the `NodeLabels` value for each AKS node pool (`AzureManagedMachinePool`) that you define in your spec. - -Below an example `nodeLabels` configuration is assigned to `agentpool0`, specifying that each node in the pool will add a label `dedicated : kafka` - -```yaml -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureManagedMachinePool -metadata: - name: agentpool0 -spec: - mode: System - osDiskSizeGB: 512 - sku: Standard_D2s_v3 - nodeLabels: - dedicated: kafka -``` - -### AKS Node Pool MaxPods configuration - -You can configure the `MaxPods` value for each AKS node pool (`AzureManagedMachinePool`) that you define in your spec (see [here](https://docs.microsoft.com/en-us/azure/aks/configure-azure-cni#configure-maximum---new-clusters) for the official AKS documentation). This corresponds to the kubelet `--max-pods` configuration (official kubelet configuration documentation can be found [here](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/)). - -Below an example `maxPods` configuration is assigned to `agentpool0`, specifying that each node in the pool will enforce a maximum of 24 scheduled pods: - -```yaml -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureManagedMachinePool -metadata: - name: agentpool0 -spec: - mode: System - osDiskSizeGB: 30 - sku: Standard_D2s_v3 - maxPods: 32 -``` - -### AKS Node Pool OsDiskType configuration - -You can configure the `OsDiskType` value for each AKS node pool (`AzureManagedMachinePool`) that you define in your spec (see [here](https://docs.microsoft.com/en-us/azure/aks/cluster-configuration#ephemeral-os) for the official AKS documentation). There are two options to choose from: `"Managed"` (the default) or `"Ephemeral"`. - -Below an example `osDiskType` configuration is assigned to `agentpool0`, specifying that each node in the pool will use a local, ephemeral OS disk for faster disk I/O at the expense of possible data loss: - -```yaml -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureManagedMachinePool -metadata: - name: agentpool0 -spec: - mode: System - osDiskSizeGB: 30 - sku: Standard_D2s_v3 - osDiskType: "Ephemeral" -``` - -## AKS Node Pool KubeletDiskType configuration - -You can configure the `KubeletDiskType` value for each AKS node pool (`AzureManagedMachinePool`) that you define in your spec (see [here](https://learn.microsoft.com/en-us/rest/api/aks/agent-pools/create-or-update?tabs=HTTP#kubeletdisktype) for the official AKS documentation). There are two options to choose from: `"OS"` or `"Temporary"`. - -Before this feature can be used, you must register the `KubeletDisk` feature on your Azure subscription with the following az cli command. - -```bash -az feature register --namespace Microsoft.ContainerService --name KubeletDisk -``` - -Below an example `kubeletDiskType` configuration is assigned to `agentpool0`, specifying that the emptyDir volumes, container runtime data root, and Kubelet ephemeral storage will be stored on the temporary disk: - -```yaml -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureManagedMachinePool -metadata: - name: agentpool0 -spec: - mode: System - osDiskSizeGB: 30 - sku: Standard_D2s_v3 - kubeletDiskType: "Temporary" -``` - -### AKS Node Pool Taints - -You can configure the `Taints` value for each AKS node pool (`AzureManagedMachinePool`) that you define in your spec. - -Below is an example of `taints` configuration for the `agentpool0`: - -```yaml -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureManagedMachinePool -metadata: - name: agentpool0 -spec: - mode: System - osDiskSizeGB: 512 - sku: Standard_D2s_v3 - taints: - - effect: no-schedule - key: dedicated - value: kafka -``` - -### AKS Node Pool OS Type - -If your cluster uses the Azure network plugin (`AzureManagedControlPlane.networkPlugin`) you can set the operating system -for your User nodepools. The `osType` field is immutable and only can be set at creation time, it defaults to `Linux` and -can be either `Linux` or `Windows`. - -```yaml -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureManagedMachinePool -metadata: - name: agentpool0 -spec: - mode: User - osDiskSizeGB: 30 - sku: Standard_D2s_v3 - osDiskType: "Ephemeral" - osType: Windows -``` - -### AKS Node Pool Kubelet Custom Configuration - -Reference: - -- https://learn.microsoft.com/en-us/azure/aks/custom-node-configuration - -When you create your node pool (`AzureManagedMachinePool`), you may specify various kubelet configuration which tunes the kubelet runtime on all nodes in that pool. For example: - -```yaml -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureManagedMachinePool -metadata: - name: pool1 -spec: - mode: User - kubeletConfig: - cpuManagerPolicy: "static" - cpuCfsQuota: true - cpuCfsQuotaPeriod: "110ms" - imageGcHighThreshold: 70 - imageGcLowThreshold: 50 - topologyManagerPolicy: "best-effort" - allowedUnsafeSysctls: - - "net.*" - - "kernel.msg*" - failSwapOn: false - containerLogMaxSizeMB: 500 - containerLogMaxFiles: 50 - podMaxPids: 2048 -``` - -Below are the full set of AKS-supported kubeletConfig configurations. All properties are children of the `spec.kubeletConfig` configuration in an `AzureManagedMachinePool` resource: - -| Configuration | Property Type | Allowed Value(s) | -|-----------------------------|-------------------|------------------------------------------------------------------------------------------| -| `cpuManagerPolicy` | string | `"none"`, `"static"` | -| `cpuCfsQuota` | boolean | `true`, `false` | -| `cpuCfsQuotaPeriod` | string | value in milliseconds, must end in `"ms"`, e.g., `"100ms"` | -| `failSwapOn` | boolean | `true`, `false` | -| `imageGcHighThreshold` | integer | integer values in the range 0-100 (inclusive) | -| `imageGcLowThreshold` | integer | integer values in the range 0-100 (inclusive), must be lower than `imageGcHighThreshold` | -| `topologyManagerPolicy` | string | `"none"`, `"best-effort"`, `"restricted"`, `"single-numa-node"` | -| `allowedUnsafeSysctls` | string | `"kernel.shm*"`, `"kernel.msg*"`, `"kernel.sem"`, `"fs.mqueue.*"`, `"net.*"` | -| `containerLogMaxSizeMB` | integer | any integer | -| `containerLogMaxFiles` | integer | any integer >= `2` | -| `podMaxPids` | integer | any integer >= `-1`, note that this must not be higher than kernel PID limit | - -For more detailed information on the behaviors of the above configurations, see [the official Kubernetes documentation](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/). Note that not all possible Kubernetes Kubelet Configuration options are available to use on your AKS node pool, only those specified above. - -CAPZ will not assign any default values for any excluded configuration properties. It is also not required to include the `spec.kubeletConfig` configuration in an `AzureManagedMachinePool` resource spec. In cases where no CAPZ configuration is declared, AKS will apply its own opinionated default configurations when the node pool is created. - -Note: these configurations can not be updated after a node pool is created. - ### Enable AKS features with custom headers (--aks-custom-headers) To enable some AKS cluster / node pool features you need to pass special headers to the cluster / node pool create request. -For example, to [add a node pool for GPU nodes](https://docs.microsoft.com/en-us/azure/aks/gpu-cluster#add-a-node-pool-for-gpu-nodes), +For example, to [add a node pool for GPU nodes](https://learn.microsoft.com/azure/aks/gpu-cluster#add-a-node-pool-for-gpu-nodes), you need to pass a custom header `UseGPUDedicatedVHD=true` (with `--aks-custom-headers UseGPUDedicatedVHD=true` argument). To do this with CAPZ, you need to add special annotations to AzureManagedCluster (for cluster features) or AzureManagedMachinePool (for node pool features). These annotations should have a prefix `infrastructure.cluster.x-k8s.io/custom-header-` followed @@ -558,194 +265,6 @@ spec: ... ``` -### Use a public Standard Load Balancer - -A public Load Balancer when integrated with AKS serves two purposes: - -- To provide outbound connections to the cluster nodes inside the AKS virtual network. It achieves this objective by translating the nodes private IP address to a public IP address that is part of its Outbound Pool. -- To provide access to applications via Kubernetes services of type LoadBalancer. With it, you can easily scale your applications and create highly available services. - -For more documentation about public Standard Load Balancer refer [AKS Doc](https://docs.microsoft.com/en-us/azure/aks/load-balancer-standard) and [AKS REST API Doc](https://docs.microsoft.com/en-us/rest/api/aks/managed-clusters/create-or-update) - -```yaml -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureManagedControlPlane -metadata: - name: my-cluster-control-plane -spec: - location: southcentralus - resourceGroupName: foo-bar - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - subscriptionID: 00000000-0000-0000-0000-000000000000 # fake uuid - version: v1.21.2 - loadBalancerProfile: # Load balancer profile must specify at most one of ManagedOutboundIPs, OutboundIPPrefixes and OutboundIPs - managedOutboundIPs: 2 # 1-100 - outboundIPPrefixes: - - /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/foo-bar/providers/Microsoft.Network/publicIPPrefixes/my-public-ip-prefix # fake public ip prefix - outboundIPs: - - /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/foo-bar/providers/Microsoft.Network/publicIPAddresses/my-public-ip # fake public ip - allocatedOutboundPorts: 100 # 0-64000 - idleTimeoutInMinutes: 10 # 4-120 -``` - -### Secure access to the API server using authorized IP address ranges - -In Kubernetes, the API server receives requests to perform actions in the cluster such as to create resources or scale the number of nodes. The API server is the central way to interact with and manage a cluster. To improve cluster security and minimize attacks, the API server should only be accessible from a limited set of IP address ranges. - -For more documentation about authorized IP address ranges refer [AKS Doc](https://docs.microsoft.com/en-us/azure/aks/api-server-authorized-ip-ranges) and [AKS REST API Doc](https://docs.microsoft.com/en-us/rest/api/aks/managed-clusters/create-or-update) - -```yaml -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureManagedControlPlane -metadata: - name: my-cluster-control-plane -spec: - location: southcentralus - resourceGroupName: foo-bar - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - subscriptionID: 00000000-0000-0000-0000-000000000000 # fake uuid - version: v1.21.2 - apiServerAccessProfile: - authorizedIPRanges: - - 12.34.56.78/32 - enablePrivateCluster: false - privateDNSZone: None # System, None. Allowed only when enablePrivateCluster is true - enablePrivateClusterPublicFQDN: false # Allowed only when enablePrivateCluster is true -``` - -### OS configurations of Linux agent nodes (AKS) - -Reference: - -- [How-to-guide Linux OS Custom Configuration](https://learn.microsoft.com/en-us/azure/aks/custom-node-configuration#linux-os-custom-configuration) -- [AKS API definition Linux OS Config](https://learn.microsoft.com/en-us/rest/api/aks/agent-pools/create-or-update?tabs=HTTP#linuxosconfig) - -When you create your node pool (`AzureManagedMachinePool`), you can specify configuration which tunes the linux OS configuration on all nodes in that pool. For example: - -```yaml -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureManagedMachinePool -metadata: - name: "${CLUSTER_NAME}-pool1" -spec: - linuxOSConfig: - swapFileSizeMB: 1500 - sysctls: - fsAioMaxNr: 65536 - fsFileMax: 8192 - fsInotifyMaxUserWatches: 781250 - fsNrOpen: 8192 - kernelThreadsMax: 20 - netCoreNetdevMaxBacklog: 1000 - netCoreOptmemMax: 20480 - netCoreRmemDefault: 212992 - netCoreRmemMax: 212992 - netCoreSomaxconn: 163849 - netCoreWmemDefault: 212992 - netCoreWmemMax: 212992 - netIpv4IPLocalPortRange: "32000 60000" - netIpv4NeighDefaultGcThresh1: 128 - netIpv4NeighDefaultGcThresh2: 512 - netIpv4NeighDefaultGcThresh3: 1024 - netIpv4TCPFinTimeout: 5 - netIpv4TCPKeepaliveProbes: 1 - netIpv4TCPKeepaliveTime: 30 - netIpv4TCPMaxSynBacklog: 128 - netIpv4TCPMaxTwBuckets: 8000 - netIpv4TCPTwReuse: true - netIpv4TCPkeepaliveIntvl: 10 - netNetfilterNfConntrackBuckets: 65536 - netNetfilterNfConntrackMax: 131072 - vmMaxMapCount: 65530 - vmSwappiness: 10 - vmVfsCachePressure: 15 - transparentHugePageDefrag: "defer+madvise" - transparentHugePageEnabled: "madvise" -``` - -Below are the full set of AKS-supported `linuxOSConfig` configurations. All properties are children of the `spec.linuxOSConfig` configuration in an `AzureManagedMachinePool` resource: - -| Configuration | Property Type | Allowed Value(s) | -|-----------------------------|-------------------|------------------------------------------------------------------------------------------| -| `swapFileSizeMB` | integer | minimum value `1`. | -| `sysctls` | SysctlConfig | | -| `transparentHugePageDefrag` | string | `"always"`, `"defer"`, `"defer+madvise"`, `"madvise"` or `"never"` | -| `transparentHugePageEnabled`| string | `"always"`, `"madvise"` or `"never"` | - -**Note**: To enable swap file on nodes, i.e.`swapFileSizeMB` to be applied, `Kubeletconfig.failSwapOn` must be set to `false` - -#### SysctlsConfig - -Below are the full set of supported `SysctlConfig` configurations. All properties are children of the `spec.linuxOSConfig.sysctls` configuration in an `AzureManagedMachinePool` resource: - -| Configuration | Property Type | Allowed Value(s) | -|---------------------------------|-------------------|------------------------------------------------------------------------------------------| -| `fsAioMaxNr` | integer | allowed value in the range [`65536` - `6553500`] (inclusive) | -| `fsFileMax` | integer | allowed value in the range [`8192` - `12000500`] (inclusive) | -| `fsInotifyMaxUserWatches` | integer | allowed value in the range [`781250` - `2097152`] (inclusive) | -| `fsNrOpen` | integer | allowed value in the range [`8192` - `20000500`] (inclusive) | -| `kernelThreadsMax` | integer | allowed value in the range [`20` - `513785`] (inclusive) | -| `netCoreNetdevMaxBacklog` | integer | allowed value in the range [`1000` - `3240000`] (inclusive) | -| `netCoreOptmemMax` | integer | allowed value in the range [`20480` - `4194304`] (inclusive) | -| `netCoreRmemDefault` | integer | allowed value in the range [`212992` - `134217728`] (inclusive) | -| `netCoreRmemMax` | integer | allowed value in the range [`212992` - `134217728`] (inclusive) | -| `netCoreSomaxconn` | integer | allowed value in the range [`4096` - `3240000`] (inclusive) | -| `netCoreWmemDefault` | integer | allowed value in the range [`212992` - `134217728`] (inclusive) | -| `netCoreWmemMax` | integer | allowed value in the range [`212992`- `134217728`] (inclusive) | -| `netIpv4IPLocalPortRange` | string | Must be specified as `"first last"`. Ex: `1024 33000`. First must be in `[1024 - 60999]` and last must be in `[32768 - 65000]`| -| `netIpv4NeighDefaultGcThresh1` | integer | allowed value in the range [`128` - `80000`] (inclusive) | -| `netIpv4NeighDefaultGcThresh2` | integer | allowed value in the range [`512` - `90000`] (inclusive) | -| `netIpv4NeighDefaultGcThresh3` | integer | allowed value in the range [`1024` - `100000`] (inclusive) | -| `netIpv4TCPFinTimeout` | integer | allowed value in the range [`5` - `120`] (inclusive) | -| `netIpv4TCPKeepaliveProbes` | integer | allowed value in the range [`1` - `15`] (inclusive) | -| `netIpv4TCPKeepaliveTime` | integer | allowed value in the range [`30` - `432000`] (inclusive) | -| `netIpv4TCPMaxSynBacklog` | integer | allowed value in the range [`128` - `3240000`] (inclusive) | -| `netIpv4TCPMaxTwBuckets` | integer | allowed value in the range [`8000` - `1440000`] (inclusive) | -| `netIpv4TCPTwReuse` | bool | allowed values `true` or `false` | -| `netIpv4TCPkeepaliveIntvl` | integer | allowed value in the range [`1` - `75`] (inclusive) | -| `netNetfilterNfConntrackBuckets`| integer | allowed value in the range [`65536` - `147456`] (inclusive) | -| `netNetfilterNfConntrackMax` | integer | allowed value in the range [`131072` - `1048576`] (inclusive) | -| `vmMaxMapCount` | integer | allowed value in the range [`65530` - `262144`] (inclusive) | -| `vmSwappiness` | integer | allowed value in the range [`0` - `100`] (inclusive) | -| `vmVfsCachePressure` | integer | allowed value in the range [`1` - `500`] (inclusive) | - -**Note**: Both of the values must be specified to enforce `NetIpv4IPLocalPortRange`. - -## Immutable fields for Managed Clusters (AKS) - -Some fields from the family of Managed Clusters CRD are immutable. Which means -those can only be set during the creation time. - -Following is the list of immutable fields for managed clusters: - -| CRD | jsonPath | Comment | -|---------------------------|------------------------------|---------------------------| -| AzureManagedCluster | .spec.controlPlaneEndpoint | populated by the AKS API during cluster create | -| AzureManagedControlPlane | .spec.controlPlaneEndpoint | populated by the AKS API during cluster create | -| AzureManagedControlPlane | .spec.subscriptionID | | -| AzureManagedControlPlane | .spec.resourceGroupName | | -| AzureManagedControlPlane | .spec.nodeResourceGroupName | | -| AzureManagedControlPlane | .spec.location | | -| AzureManagedControlPlane | .spec.sshPublicKey | | -| AzureManagedControlPlane | .spec.dnsServiceIP | | -| AzureManagedControlPlane | .spec.networkPlugin | | -| AzureManagedControlPlane | .spec.networkPolicy | | -| AzureManagedControlPlane | .spec.loadBalancerSKU | | -| AzureManagedControlPlane | .spec.apiServerAccessProfile | except AuthorizedIPRanges | -| AzureManagedControlPlane | .spec.virtualNetwork | | -| AzureManagedControlPlane | .spec.virtualNetwork.subnet | except serviceEndpoints | -| AzureManagedMachinePool | .spec.name | | -| AzureManagedMachinePool | .spec.sku | | -| AzureManagedMachinePool | .spec.osDiskSizeGB | | -| AzureManagedMachinePool | .spec.osDiskType | | -| AzureManagedMachinePool | .spec.availabilityZones | | -| AzureManagedMachinePool | .spec.maxPods | | -| AzureManagedMachinePool | .spec.osType | | -| AzureManagedMachinePool | .spec.enableNodePublicIP | | -| AzureManagedMachinePool | .spec.nodePublicIPPrefixID | | -| AzureManagedMachinePool | .spec.kubeletConfig | | -| AzureManagedMachinePool | .spec.linuxOSConfig | | - ## Features AKS clusters deployed from CAPZ currently only support a limited, @@ -766,6 +285,10 @@ Current limitations - Only supports Azure Active Directory Managed by Azure. - We will not support Legacy Azure Active Directory +## Best Practices + +A set of best practices for managing AKS clusters is documented here: https://learn.microsoft.com/azure/aks/best-practices + ## Troubleshooting If a user tries to delete the MachinePool which refers to the last system node pool AzureManagedMachinePool webhook will reject deletion, so time stamp never gets set on the AzureManagedMachinePool. However the timestamp would be set on the MachinePool and would be in deletion state. To recover from this state create a new MachinePool manually referencing the AzureManagedMachinePool, edit the required references and finalizers to link the MachinePool to the AzureManagedMachinePool. In the AzureManagedMachinePool remove the owner reference to the old MachinePool, and set it to the new MachinePool. Once the new MachinePool is pointing to the AzureManagedMachinePool you can delete the old MachinePool. To delete the old MachinePool remove the finalizers in that object.