Merge pull request #32 from msfidelis/feature/upgrades

tests: karpenter v1
msfidelis · Oct 11, 2024 · 63a64d1 · 63a64d1
2 parents f0d4655 + a5d55ef
commit 63a64d1
Show file tree

Hide file tree

Showing 10 changed files with 130 additions and 75 deletions.
diff --git a/helm/karpenter/templates/node-class.yaml.tpl b/helm/karpenter/templates/node-class.yaml.tpl
@@ -0,0 +1,29 @@
+apiVersion: karpenter.k8s.aws/v1
+kind: EC2NodeClass
+metadata:
+  name: ${EKS_CLUSTER}-nodeclass
+spec:
+
+  instanceProfile: ${INSTANCE_PROFILE}
+  amiSelectorTerms:
+
+  - id: ${AMI_ID}
+
+  amiFamily: ${AMI_FAMILY}
+
+  securityGroupSelectorTerms:
+  %{ for sg in SECURITY_GROUPS ~}
+
+  - id: ${sg}
+
+  %{ endfor ~}  
+
+  subnetSelectorTerms:
+  %{ for sb in EKS_SUBNETS ~}
+
+  - id: ${sb}
+
+  %{ endfor ~}  
+
+  launchTemplate:
+    id: ${LAUNCH_TEMPLATE}
diff --git a/helm/karpenter/templates/node-pool.yaml.tpl b/helm/karpenter/templates/node-pool.yaml.tpl
@@ -0,0 +1,55 @@
+apiVersion: karpenter.sh/v1
+kind: NodePool
+metadata:
+  name: ${EKS_CLUSTER}-nodepool
+spec:
+  disruption:
+    consolidationPolicy: WhenEmptyOrUnderutilized
+    consolidateAfter: Never
+  topologySpreadConstraints:
+    - maxSkew: 1
+      topologyKey: "topology.kubernetes.io/zone"
+      whenUnsatisfiable: ScheduleAnyway
+  template:
+    spec:
+      requirements:
+        - key: karpenter.k8s.aws/instance-family
+          operator: In
+          values: 
+  %{ for ifm in INSTANCE_FAMILY ~}
+
+          - ${ifm}
+
+  %{ endfor ~}
+
+        - key: karpenter.sh/capacity-type
+          operator: In
+          values:
+  %{ for cpct in CAPACITY_TYPE ~}
+
+          - ${cpct}
+
+  %{ endfor ~}
+
+        - key: karpenter.k8s.aws/instance-size
+          operator: In
+          values:
+  %{ for ifs in INSTANCE_SIZES ~}
+
+          - ${ifs}
+
+  %{ endfor ~}       
+
+        - key: "topology.kubernetes.io/zone" 
+          operator: In
+          values:
+  %{ for az in AVAILABILITY_ZONES ~}
+
+          - ${az}
+
+  %{ endfor ~}
+
+      nodeClassRef:
+        group: karpenter.k8s.aws
+        kind: EC2NodeClass
+        name: ${EKS_CLUSTER}-nodeclass
diff --git a/helm/karpenter/templates/provisioner.yml.tpl b/helm/karpenter/templates/provisioner.yml.tpl
diff --git a/helm/karpenter/templates/template.yml.tpl b/helm/karpenter/templates/template.yml.tpl
diff --git a/helm_jaeger.tf b/helm_jaeger.tf
@@ -1,4 +1,7 @@
 resource "helm_release" "jaeger" {
+
+  count = var.enable_jaeger ? 1 : 0
+
   name       = "jaeger"
   repository = "https://jaegertracing.github.io/helm-charts"
   chart      = "jaeger"
@@ -16,6 +19,9 @@ resource "helm_release" "jaeger" {
 }
 
 resource "kubectl_manifest" "jaeger_gateway" {
+
+  count = var.enable_jaeger ? 1 : 0
+
   yaml_body = <<YAML
 apiVersion: networking.istio.io/v1alpha3
 kind: Gateway
@@ -46,6 +52,9 @@ YAML
 }
 
 resource "kubectl_manifest" "jaeger_virtual_service" {
+
+  count = var.enable_jaeger ? 1 : 0
+
   yaml_body = <<YAML
 apiVersion: networking.istio.io/v1alpha3
 kind: VirtualService

diff --git a/helm_karpenter.tf b/helm_karpenter.tf
@@ -4,22 +4,22 @@ resource "helm_release" "karpenter" {
   create_namespace = true
 
   name       = "karpenter"
-  repository = "https://charts.karpenter.sh"
+  repository = "oci://public.ecr.aws/karpenter"
   chart      = "karpenter"
-  version    = "v0.15.0"
+  version    = "1.0.5"
 
   set {
     name  = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn"
     value = aws_iam_role.karpenter_role[count.index].arn
   }
 
   set {
-    name  = "clusterName"
+    name  = "settings.clusterName"
     value = var.cluster_name
   }
 
   set {
-    name  = "clusterEndpoint"
+    name  = "settings.clusterEndpoint"
     value = aws_eks_cluster.main.endpoint
   }
 
@@ -36,10 +36,10 @@ resource "helm_release" "karpenter" {
 
 }
 
-resource "kubectl_manifest" "karpenter_provisioner" {
+resource "kubectl_manifest" "karpenter_node_pool" {
   count = var.karpenter_toggle ? 1 : 0
   yaml_body = templatefile(
-    "${path.module}/helm/karpenter/templates/provisioner.yml.tpl", {
+    "${path.module}/helm/karpenter/templates/node-pool.yaml.tpl", {
       EKS_CLUSTER        = var.cluster_name,
       CAPACITY_TYPE      = var.karpenter_capacity_type
       INSTANCE_FAMILY    = var.karpenter_instance_family
@@ -52,16 +52,24 @@ resource "kubectl_manifest" "karpenter_provisioner" {
   ]
 }
 
-resource "kubectl_manifest" "karpenter_template" {
+resource "kubectl_manifest" "karpenter_node_class" {
   count = var.karpenter_toggle ? 1 : 0
   yaml_body = templatefile(
-    "${path.module}/helm/karpenter/templates/template.yml.tpl", {
-      EKS_CLUSTER = var.cluster_name,
-      EKS_SUBNETS = join(", ", [
+    "${path.module}/helm/karpenter/templates/node-class.yaml.tpl", {
+      EKS_CLUSTER = var.cluster_name
+      AMI_FAMILY = var.karpenter_ec2_node_family
+      SECURITY_GROUPS = [
+        aws_security_group.cluster_sg.id,
+        aws_security_group.cluster_nodes_sg.id,
+        aws_eks_cluster.main.vpc_config[0].cluster_security_group_id
+      ]
+      INSTANCE_PROFILE = aws_iam_instance_profile.nodes.name
+      AMI_ID = data.aws_ssm_parameter.eks.value
+      EKS_SUBNETS = [
         aws_subnet.private_subnet_1a.id,
         aws_subnet.private_subnet_1b.id,
         aws_subnet.private_subnet_1c.id
-      ])
+      ]
       LAUNCH_TEMPLATE = format("%s-karpenter", var.cluster_name)
   })
 

diff --git a/iam_karpenter.tf b/iam_karpenter.tf
@@ -39,12 +39,15 @@ data "aws_iam_policy_document" "karpenter_policy" {
       "ec2:DescribeInstances",
       "ec2:DescribeSecurityGroups",
       "ec2:DescribeSubnets",
+      "ec2:DescribeImages",
       "ec2:DescribeInstanceTypes",
       "ec2:DescribeInstanceTypeOfferings",
       "ec2:DescribeAvailabilityZones",
       "ec2:DescribeSpotPriceHistory",
       "pricing:GetProducts",
       "ec2:RunInstances",
+      "ec2:TerminateInstances",
+      "ec2:DeleteLaunchTemplate",
       "ssm:GetParameter",
       "iam:PassRole"
     ]
@@ -55,19 +58,6 @@ data "aws_iam_policy_document" "karpenter_policy" {
 
   }
 
-  statement {
-
-    effect = "Allow"
-    actions = [
-      "ec2:TerminateInstances",
-      "ec2:DeleteLaunchTemplate",
-    ]
-
-    resources = [
-      "*"
-    ]
-
-  }
 
 }
 

diff --git a/provider.tf b/provider.tf
@@ -49,4 +49,5 @@ provider "kubectl" {
   host                   = aws_eks_cluster.main.endpoint
   cluster_ca_certificate = base64decode(aws_eks_cluster.main.certificate_authority.0.data)
   token                  = data.aws_eks_cluster_auth.default.token
+  apply_retry_count      = 15
 }
diff --git a/samples/chip/chip-istio.yml b/samples/chip/chip-istio.yml
@@ -13,7 +13,7 @@ metadata:
   name: chip
   namespace: chip
 spec:
-  replicas: 2
+  replicas: 200
   selector:
     matchLabels:
       app: chip
@@ -94,8 +94,8 @@ metadata:
   name: chip
   namespace: chip
 spec:
-  maxReplicas: 10
-  minReplicas: 2
+  maxReplicas: 1000
+  minReplicas: 200
   metrics:
     - type: Resource
       resource:

diff --git a/variables.tf b/variables.tf
@@ -91,6 +91,10 @@ variable "karpenter_availability_zones" {
   ]
 }
 
+variable "karpenter_ec2_node_family" {
+  default = "Bottlerocket"
+}
+
 #########################
 ###  INGRESS CONFIGS  ###
 #########################
@@ -128,7 +132,7 @@ variable "enable_cross_zone_load_balancing" {
 variable "enable_vpc_link" {
   type        = bool
   description = "Create VPC Link associated to Network Load Balancing"
-  default     = true
+  default     = false
 }
 
 #########################
@@ -179,6 +183,12 @@ variable "kiali_virtual_service_host" {
   default     = "kiali.k8s.raj.ninja"
 }
 
+variable "enable_jaeger" {
+  type = bool
+  description = "Flag to create jaeger standalone stack"
+  default = false
+}
+
 variable "jaeger_virtual_service_host" {
   type        = string
   description = "The hostname for the Jaeger virtual service, used for tracing and monitoring microservices within the Istio service mesh."