From 38a773fb6b54956c967b9f1e75cbc6385d32fdda Mon Sep 17 00:00:00 2001 From: gongmax Date: Fri, 27 Oct 2023 18:20:57 +0000 Subject: [PATCH 1/4] Disable deletion protection for Autopilot test clusters --- .../modules/gke-autopilot/cluster.tf | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/install/terraform/modules/gke-autopilot/cluster.tf b/install/terraform/modules/gke-autopilot/cluster.tf index 050aa5c2dc..ec21dc53cb 100644 --- a/install/terraform/modules/gke-autopilot/cluster.tf +++ b/install/terraform/modules/gke-autopilot/cluster.tf @@ -22,15 +22,15 @@ data "google_client_config" "default" {} # A list of all parameters used in interpolation var.cluster # Set values to default if not key was not set in original map locals { - name = lookup(var.cluster, "name", "test-cluster") - project = lookup(var.cluster, "project", "agones") - location = lookup(var.cluster, "location", "us-west1") - network = lookup(var.cluster, "network", "default") - subnetwork = lookup(var.cluster, "subnetwork", "") - releaseChannel = lookup(var.cluster, "releaseChannel", "REGULAR") - kubernetesVersion = lookup(var.cluster, "kubernetesVersion", "1.27") - maintenanceExclusionStartTime = lookup(var.cluster, "maintenanceExclusionStartTime", timestamp()) - maintenanceExclusionEndTime = lookup(var.cluster, "maintenanceExclusionEndTime", timeadd(timestamp(), "4080h")) # 170 days + name = lookup(var.cluster, "name", "test-cluster") + project = lookup(var.cluster, "project", "agones") + location = lookup(var.cluster, "location", "us-west1") + network = lookup(var.cluster, "network", "default") + subnetwork = lookup(var.cluster, "subnetwork", "") + releaseChannel = lookup(var.cluster, "releaseChannel", "REGULAR") + kubernetesVersion = lookup(var.cluster, "kubernetesVersion", "1.27") + maintenanceExclusionStartTime = lookup(var.cluster, "maintenanceExclusionStartTime", timestamp()) + maintenanceExclusionEndTime = lookup(var.cluster, "maintenanceExclusionEndTime", timeadd(timestamp(), "4080h")) # 170 days } # echo command used for debugging purpose @@ -54,11 +54,12 @@ resource "null_resource" "test-setting-variables" { resource "google_container_cluster" "primary" { provider = google-beta # required for node_pool_auto_config.network_tags - name = local.name - project = local.project - location = local.location - network = local.network - subnetwork = local.subnetwork + name = local.name + project = local.project + location = local.location + network = local.network + subnetwork = local.subnetwork + deletion_protection = false release_channel { channel = local.releaseChannel != "" ? local.releaseChannel : "UNSPECIFIED" @@ -69,11 +70,11 @@ resource "google_container_cluster" "primary" { # When exclusions and maintenance windows overlap, exclusions have precedence. daily_maintenance_window { start_time = "03:00" - } - maintenance_exclusion{ + } + maintenance_exclusion { exclusion_name = format("%s-%s", local.name, "exclusion") - start_time = local.maintenanceExclusionStartTime - end_time = local.maintenanceExclusionEndTime + start_time = local.maintenanceExclusionStartTime + end_time = local.maintenanceExclusionEndTime exclusion_options { scope = "NO_MINOR_UPGRADES" } From efe6b8e44d06a0bfb828b9775c4a3b41eb8f0be2 Mon Sep 17 00:00:00 2001 From: gongmax Date: Mon, 30 Oct 2023 17:11:14 +0000 Subject: [PATCH 2/4] fix --- build/terraform/gke-autopilot/module.tf | 9 +++++---- install/terraform/modules/gke-autopilot/cluster.tf | 3 ++- install/terraform/modules/gke-autopilot/variables.tf | 1 + 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/build/terraform/gke-autopilot/module.tf b/build/terraform/gke-autopilot/module.tf index 0aa7aac8e1..68235aa807 100644 --- a/build/terraform/gke-autopilot/module.tf +++ b/build/terraform/gke-autopilot/module.tf @@ -99,10 +99,11 @@ module "gke_autopilot_cluster" { source = "../../../install/terraform/modules/gke-autopilot" cluster = { - "name" = var.name - "project" = var.project - "location" = var.location - "network" = var.network + "name" = var.name + "project" = var.project + "location" = var.location + "network" = var.network + "deleteProtection" = false } } diff --git a/install/terraform/modules/gke-autopilot/cluster.tf b/install/terraform/modules/gke-autopilot/cluster.tf index ec21dc53cb..55690b544a 100644 --- a/install/terraform/modules/gke-autopilot/cluster.tf +++ b/install/terraform/modules/gke-autopilot/cluster.tf @@ -31,6 +31,7 @@ locals { kubernetesVersion = lookup(var.cluster, "kubernetesVersion", "1.27") maintenanceExclusionStartTime = lookup(var.cluster, "maintenanceExclusionStartTime", timestamp()) maintenanceExclusionEndTime = lookup(var.cluster, "maintenanceExclusionEndTime", timeadd(timestamp(), "4080h")) # 170 days + deletionProtection = lookup(var.cluster, "deletionProtection", true) } # echo command used for debugging purpose @@ -59,7 +60,7 @@ resource "google_container_cluster" "primary" { location = local.location network = local.network subnetwork = local.subnetwork - deletion_protection = false + deletion_protection = local.deletionProtection release_channel { channel = local.releaseChannel != "" ? local.releaseChannel : "UNSPECIFIED" diff --git a/install/terraform/modules/gke-autopilot/variables.tf b/install/terraform/modules/gke-autopilot/variables.tf index 27c9b5500c..3936809e83 100644 --- a/install/terraform/modules/gke-autopilot/variables.tf +++ b/install/terraform/modules/gke-autopilot/variables.tf @@ -27,6 +27,7 @@ variable "cluster" { "subnetwork" = "" "releaseChannel" = "REGULAR" "kubernetesVersion" = "1.27" + "deleteProtection" = true } } From 89eaf73b478325df43819e1b4f3d9f4dd992b513 Mon Sep 17 00:00:00 2001 From: gongmax Date: Mon, 30 Oct 2023 17:15:04 +0000 Subject: [PATCH 3/4] fix --- build/terraform/gke-autopilot/module.tf | 10 +++++----- .../terraform/modules/gke-autopilot/variables.tf | 16 ++++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/build/terraform/gke-autopilot/module.tf b/build/terraform/gke-autopilot/module.tf index 68235aa807..373a2a3591 100644 --- a/build/terraform/gke-autopilot/module.tf +++ b/build/terraform/gke-autopilot/module.tf @@ -99,11 +99,11 @@ module "gke_autopilot_cluster" { source = "../../../install/terraform/modules/gke-autopilot" cluster = { - "name" = var.name - "project" = var.project - "location" = var.location - "network" = var.network - "deleteProtection" = false + "name" = var.name + "project" = var.project + "location" = var.location + "network" = var.network + "deletionProtection" = false } } diff --git a/install/terraform/modules/gke-autopilot/variables.tf b/install/terraform/modules/gke-autopilot/variables.tf index 3936809e83..13787f8852 100644 --- a/install/terraform/modules/gke-autopilot/variables.tf +++ b/install/terraform/modules/gke-autopilot/variables.tf @@ -20,14 +20,14 @@ variable "cluster" { type = map(any) default = { - "name" = "test-cluster" - "project" = "agones" - "location" = "us-west1" - "network" = "default" - "subnetwork" = "" - "releaseChannel" = "REGULAR" - "kubernetesVersion" = "1.27" - "deleteProtection" = true + "name" = "test-cluster" + "project" = "agones" + "location" = "us-west1" + "network" = "default" + "subnetwork" = "" + "releaseChannel" = "REGULAR" + "kubernetesVersion" = "1.27" + "deletionProtection" = true } } From 9c8a539a87cf12f858f678c8cb67ce5f1cc70041 Mon Sep 17 00:00:00 2001 From: gongmax Date: Thu, 14 Dec 2023 21:35:58 +0000 Subject: [PATCH 4/4] Add option to disable deletion protection for AP test cluster, and make maintainance exclusive window optional --- build/terraform/e2e/gke-autopilot/module.tf | 13 +++++--- build/terraform/e2e/gke-standard/module.tf | 18 ++++++----- build/terraform/gke-autopilot/module.tf | 9 +++--- .../modules/gke-autopilot/cluster.tf | 29 +++++++++-------- .../modules/gke-autopilot/variables.tf | 18 ++++++----- install/terraform/modules/gke/cluster.tf | 9 +++--- install/terraform/modules/gke/variables.tf | 32 ++++++++++--------- 7 files changed, 69 insertions(+), 59 deletions(-) diff --git a/build/terraform/e2e/gke-autopilot/module.tf b/build/terraform/e2e/gke-autopilot/module.tf index d29e3517de..da24c3cd90 100644 --- a/build/terraform/e2e/gke-autopilot/module.tf +++ b/build/terraform/e2e/gke-autopilot/module.tf @@ -39,11 +39,14 @@ module "gke_cluster" { source = "../../../../install/terraform/modules/gke-autopilot" cluster = { - "name" = format("gke-autopilot-e2e-test-cluster-%s", replace(var.kubernetesVersion, ".", "-")) - "project" = var.project - "location" = var.location - "releaseChannel" = var.releaseChannel - "kubernetesVersion" = var.kubernetesVersion + "name" = format("gke-autopilot-e2e-test-cluster-%s", replace(var.kubernetesVersion, ".", "-")) + "project" = var.project + "location" = var.location + "releaseChannel" = var.releaseChannel + "kubernetesVersion" = var.kubernetesVersion + "deletionProtection" = false + "maintenanceExclusionStartTime" = timestamp() + "maintenanceExclusionEndTime" = timeadd(timestamp(), "2640h") # 110 days } udpFirewall = false // firewall is created at the project module level diff --git a/build/terraform/e2e/gke-standard/module.tf b/build/terraform/e2e/gke-standard/module.tf index 64a2de2305..906a1a71df 100644 --- a/build/terraform/e2e/gke-standard/module.tf +++ b/build/terraform/e2e/gke-standard/module.tf @@ -51,14 +51,16 @@ module "gke_cluster" { source = "../../../../install/terraform/modules/gke" cluster = { - "name" = var.overrideName != "" ? var.overrideName : format("standard-e2e-test-cluster-%s", replace(var.kubernetesVersion, ".", "-")) - "location" = var.location - "releaseChannel" = var.releaseChannel - "machineType" = var.machineType - "initialNodeCount" = var.initialNodeCount - "enableImageStreaming" = true - "project" = var.project - "kubernetesVersion" = var.kubernetesVersion + "name" = var.overrideName != "" ? var.overrideName : format("standard-e2e-test-cluster-%s", replace(var.kubernetesVersion, ".", "-")) + "location" = var.location + "releaseChannel" = var.releaseChannel + "machineType" = var.machineType + "initialNodeCount" = var.initialNodeCount + "enableImageStreaming" = true + "project" = var.project + "kubernetesVersion" = var.kubernetesVersion + "maintenanceExclusionStartTime" = timestamp() + "maintenanceExclusionEndTime" = timeadd(timestamp(), "2640h") # 110 days } udpFirewall = false // firewall is created at the project module level diff --git a/build/terraform/gke-autopilot/module.tf b/build/terraform/gke-autopilot/module.tf index 373a2a3591..0aa7aac8e1 100644 --- a/build/terraform/gke-autopilot/module.tf +++ b/build/terraform/gke-autopilot/module.tf @@ -99,11 +99,10 @@ module "gke_autopilot_cluster" { source = "../../../install/terraform/modules/gke-autopilot" cluster = { - "name" = var.name - "project" = var.project - "location" = var.location - "network" = var.network - "deletionProtection" = false + "name" = var.name + "project" = var.project + "location" = var.location + "network" = var.network } } diff --git a/install/terraform/modules/gke-autopilot/cluster.tf b/install/terraform/modules/gke-autopilot/cluster.tf index 55690b544a..686b889b2b 100644 --- a/install/terraform/modules/gke-autopilot/cluster.tf +++ b/install/terraform/modules/gke-autopilot/cluster.tf @@ -29,8 +29,8 @@ locals { subnetwork = lookup(var.cluster, "subnetwork", "") releaseChannel = lookup(var.cluster, "releaseChannel", "REGULAR") kubernetesVersion = lookup(var.cluster, "kubernetesVersion", "1.27") - maintenanceExclusionStartTime = lookup(var.cluster, "maintenanceExclusionStartTime", timestamp()) - maintenanceExclusionEndTime = lookup(var.cluster, "maintenanceExclusionEndTime", timeadd(timestamp(), "4080h")) # 170 days + maintenanceExclusionStartTime = lookup(var.cluster, "maintenanceExclusionStartTime", null) + maintenanceExclusionEndTime = lookup(var.cluster, "maintenanceExclusionEndTime", null) deletionProtection = lookup(var.cluster, "deletionProtection", true) } @@ -67,17 +67,20 @@ resource "google_container_cluster" "primary" { } min_master_version = local.kubernetesVersion - maintenance_policy { - # When exclusions and maintenance windows overlap, exclusions have precedence. - daily_maintenance_window { - start_time = "03:00" - } - maintenance_exclusion { - exclusion_name = format("%s-%s", local.name, "exclusion") - start_time = local.maintenanceExclusionStartTime - end_time = local.maintenanceExclusionEndTime - exclusion_options { - scope = "NO_MINOR_UPGRADES" + dynamic "maintenance_policy" { + for_each = (local.releaseChannel != "UNSPECIFIED" && local.maintenanceExclusionStartTime != null && local.maintenanceExclusionEndTime != null) ? [1] : [] + content { + # When exclusions and maintenance windows overlap, exclusions have precedence. + daily_maintenance_window { + start_time = "03:00" + } + maintenance_exclusion { + exclusion_name = format("%s-%s", local.name, "exclusion") + start_time = local.maintenanceExclusionStartTime + end_time = local.maintenanceExclusionEndTime + exclusion_options { + scope = "NO_MINOR_UPGRADES" + } } } } diff --git a/install/terraform/modules/gke-autopilot/variables.tf b/install/terraform/modules/gke-autopilot/variables.tf index 13787f8852..80c3e537e0 100644 --- a/install/terraform/modules/gke-autopilot/variables.tf +++ b/install/terraform/modules/gke-autopilot/variables.tf @@ -20,14 +20,16 @@ variable "cluster" { type = map(any) default = { - "name" = "test-cluster" - "project" = "agones" - "location" = "us-west1" - "network" = "default" - "subnetwork" = "" - "releaseChannel" = "REGULAR" - "kubernetesVersion" = "1.27" - "deletionProtection" = true + "name" = "test-cluster" + "project" = "agones" + "location" = "us-west1" + "network" = "default" + "subnetwork" = "" + "releaseChannel" = "REGULAR" + "kubernetesVersion" = "1.27" + "deletionProtection" = true + "maintenanceExclusionStartTime" = null + "maintenanceExclusionEndTime" = null } } diff --git a/install/terraform/modules/gke/cluster.tf b/install/terraform/modules/gke/cluster.tf index 7af76cd3d7..c69e6f8317 100644 --- a/install/terraform/modules/gke/cluster.tf +++ b/install/terraform/modules/gke/cluster.tf @@ -39,9 +39,8 @@ locals { workloadIdentity = lookup(var.cluster, "workloadIdentity", false) minNodeCount = lookup(var.cluster, "minNodeCount", "1") maxNodeCount = lookup(var.cluster, "maxNodeCount", "5") - maintenanceExclusionStartTime = lookup(var.cluster, "maintenanceExclusionStartTime", timestamp()) - maintenanceExclusionEndTime = lookup(var.cluster, "maintenanceExclusionEndTime", timeadd(timestamp(), "2640h")) - # 110 days + maintenanceExclusionStartTime = lookup(var.cluster, "maintenanceExclusionStartTime", null) + maintenanceExclusionEndTime = lookup(var.cluster, "maintenanceExclusionEndTime", null) } data "google_container_engine_versions" "version" { @@ -82,7 +81,7 @@ resource "google_container_cluster" "primary" { networking_mode = "VPC_NATIVE" ip_allocation_policy {} - + release_channel { channel = local.releaseChannel } @@ -90,7 +89,7 @@ resource "google_container_cluster" "primary" { min_master_version = local.kubernetesVersion dynamic "maintenance_policy" { - for_each = local.releaseChannel != "UNSPECIFIED" ? [1] : [] + for_each = (local.releaseChannel != "UNSPECIFIED" && local.maintenanceExclusionStartTime != null && local.maintenanceExclusionEndTime != null) ? [1] : [] content { # When exclusions and maintenance windows overlap, exclusions have precedence. daily_maintenance_window { diff --git a/install/terraform/modules/gke/variables.tf b/install/terraform/modules/gke/variables.tf index 55acdbaf93..5b467d570f 100644 --- a/install/terraform/modules/gke/variables.tf +++ b/install/terraform/modules/gke/variables.tf @@ -20,21 +20,23 @@ variable "cluster" { type = map(any) default = { - "location" = "us-west1-c" - "name" = "test-cluster" - "machineType" = "e2-standard-4" - "initialNodeCount" = "4" - "project" = "agones" - "network" = "default" - "subnetwork" = "" - "releaseChannel" = "UNSPECIFIED" - "kubernetesVersion" = "1.27" - "windowsInitialNodeCount" = "0" - "windowsMachineType" = "e2-standard-4" - "autoscale" = false - "workloadIdentity" = false - "minNodeCount" = "1" - "maxNodeCount" = "5" + "location" = "us-west1-c" + "name" = "test-cluster" + "machineType" = "e2-standard-4" + "initialNodeCount" = "4" + "project" = "agones" + "network" = "default" + "subnetwork" = "" + "releaseChannel" = "UNSPECIFIED" + "kubernetesVersion" = "1.27" + "windowsInitialNodeCount" = "0" + "windowsMachineType" = "e2-standard-4" + "autoscale" = false + "workloadIdentity" = false + "minNodeCount" = "1" + "maxNodeCount" = "5" + "maintenanceExclusionStartTime" = null + "maintenanceExclusionEndTime" = null } }