feat(iac): openstack kubernetes module improvements

deployment/terraform/examples/openstack-kubernetes/k3s-cluster/main.tf

@@ -1,7 +1,14 @@
 module "openstack_cogstack_infra" {
   source = "../../../modules/openstack-kubernetes-infra"
   host_instances = [
-    { name = "cogstack-k3s", is_controller = true },
+    {
+      name          = "cogstack-k3s",
+      is_controller = true,
+      # floating_ip = {
+      #   use_floating_ip = true,
+      #   address         = "10.10.10.10"
+      # }
+    },
     {
       name          = "cogstack-k3s-node-2"
       flavour       = "2cpu4ram"
@@ -11,4 +18,9 @@ module "openstack_cogstack_infra" {
   ]
   allowed_ingress_ips_cidr = var.allowed_ingress_ips_cidr
   ubuntu_immage_name       = var.ubuntu_immage_name
+  # generate_random_name_prefix = false
+  # prefix                      = "dev"
+  # network = {
+  #   network_id = "some-id"
+  # }
 }

deployment/terraform/modules/openstack-cogstack-infra/compute.tf

@@ -125,6 +125,3 @@ data "openstack_images_image_v2" "ubuntu" {
   most_recent = true
 }
 
-data "openstack_networking_secgroup_v2" "er_https_from_lbs" {
-  name = "er_https_from_lbs"
-}

deployment/terraform/modules/openstack-kubernetes-infra/README.md

@@ -35,4 +35,54 @@ module "openstack_kubernetes_cluster" {
 }
 ```
 
+## Existing Network and Floating IPs
+You can use this module with a custom OpenStack network by specifying the `network` variable. By default, it will use the network named `"external_4003"`, but you can override this with your own network name or network ID. 
+
+Using a custom OpenStack network with your own subnet allows you to improve security by keeping most nodes private. This ensures worker nodes and other internal resources are not directly exposed, reducing security risks. Only the controller node needs a floating IP to be accessible in order to use the k8s api server (eg for kubectl to work from outside the network). Using floating ips will also have the advantage of being stable, so you can destroy/create VMs without having to update anywhere the IP address is referenced by reassigning the floating ip. 
+
+To use this configuration, you will probably need to assign a floating IP to the controller node so that it is accessible. You can configure floating IP assignment per node using the `floating_ip` block within each entry in the `host_instances` variable
+
+Please see this example for using thism module with a custom network and floating IPs.
+
+```hcl
+host_instances = [
+  {
+    name = "controller"
+    is_controller = true
+    floating_ip = {
+      use_floating_ip = true
+      address         = "203.0.113.10" # Address of an existing floating_ip in openstack
+    }
+  },
+  {
+    name = "worker"
+    # Optionally also assign a floating IP here, or leave blank to keep it internal to the network   
+  }
+  network = {
+    network_id = openstack_networking_network_v2.example_network.id
+  }
+]
+
+
+resource "openstack_networking_network_v2" "example_network" {
+  name           = "dev-example-network"
+  admin_state_up = "true"
+}
+
+resource "openstack_networking_subnet_v2" "example_subnet"{
+  name       = "dev-example-subnet"
+  network_id = openstack_networking_network_v2.example_network.id
+  cidr       = "192.168.0.0/24"
+  ip_version = 4
+}
+
+resource "openstack_networking_router_v2" "example_router" {
+  name                = "test-router"
+  admin_state_up      = true
+  external_network_id = data.openstack_networking_network_v2.external_4003.id
+}
+```
+
+When using a non-default network, ensure the controller host has a floating IP so Terraform can access it for provisioning.
+
 

deployment/terraform/modules/openstack-kubernetes-infra/cloud-init-k3s-agent.yaml

@@ -37,7 +37,13 @@ runcmd:
 
   #  Run K3s
   - echo "Installing K3S"
-  - curl -sfL https://get.k3s.io | K3S_URL=https://${TF_K3S_SERVER_IP_ADDRESS}:6443 K3S_TOKEN="${TF_K3S_TOKEN}" sh -
+  - |
+    INSTALL_K3S_EXEC=""
+    TF_K3S_NODE_EXTERNAL_IP=${TF_K3S_NODE_EXTERNAL_IP}
+    if [ -n "$TF_K3S_NODE_EXTERNAL_IP" ]; then
+      INSTALL_K3S_EXEC="--node-external-ip $TF_K3S_NODE_EXTERNAL_IP"
+    fi
+    curl -sfL https://get.k3s.io | K3S_URL=https://${TF_K3S_SERVER_IP_ADDRESS}:6443 K3S_TOKEN="${TF_K3S_TOKEN}" INSTALL_K3S_EXEC="$INSTALL_K3S_EXEC" sh -
   - echo "Completed Installing K3S"
  # - sudo chmod 644 /etc/rancher/k3s/k3s.yaml
 

deployment/terraform/modules/openstack-kubernetes-infra/cloud-init-k3s-server.yaml

@@ -37,7 +37,17 @@ runcmd:
 
   #  Run K3s
   - echo "Installing K3S"
-  - sudo curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server" K3S_TOKEN="${TF_K3S_TOKEN}" sh -
+  - |
+    TF_K3S_TLS_SAN=${TF_K3S_TLS_SAN}
+    TF_K3S_NODE_EXTERNAL_IP=${TF_K3S_NODE_EXTERNAL_IP}
+    INSTALL_K3S_EXEC="server"
+    if [ -n "$TF_K3S_TLS_SAN" ]; then
+      INSTALL_K3S_EXEC="$INSTALL_K3S_EXEC --tls-san $TF_K3S_TLS_SAN"
+    fi
+    if [ -n "$TF_K3S_NODE_EXTERNAL_IP" ]; then
+      INSTALL_K3S_EXEC="$INSTALL_K3S_EXEC --node-external-ip $TF_K3S_NODE_EXTERNAL_IP"
+    fi
+    sudo curl -sfL https://get.k3s.io | K3S_TOKEN="${TF_K3S_TOKEN}" INSTALL_K3S_EXEC="$INSTALL_K3S_EXEC" sh -
   - echo "Completed Installing K3S"
   - sudo chmod 644 /etc/rancher/k3s/k3s.yaml
 

deployment/terraform/modules/openstack-kubernetes-infra/compute-keypair.tf

@@ -14,7 +14,7 @@ locals {
 }
 
 resource "openstack_compute_keypair_v2" "compute_keypair" {
-  name       = "${local.random_prefix}-cogstack_keypair"
+  name       = local.prefix != "" ? "${local.prefix}-cogstack_keypair" : "cogstack_keypair"
   public_key = local.is_using_existing_ssh_keypair ? file(var.ssh_key_pair.public_key_file) : null
 }
 
@@ -30,4 +30,4 @@ resource "local_file" "public_key" {
   content         = openstack_compute_keypair_v2.compute_keypair.public_key
   filename        = "${local.output_file_directory}/${openstack_compute_keypair_v2.compute_keypair.name}-rsa.pub"
   file_permission = "0600"
-}
+}

deployment/terraform/modules/openstack-kubernetes-infra/compute.tf

@@ -1,17 +1,18 @@
 resource "openstack_compute_instance_v2" "kubernetes_server" {
 
-  name      = "${local.random_prefix}-${local.controller_host.name}"
+  name      = local.prefix != "" ? "${local.prefix}-${local.controller_host.name}" : local.controller_host.name
   flavor_id = data.openstack_compute_flavor_v2.available_compute_flavors[local.controller_host.flavour].id
   key_pair  = openstack_compute_keypair_v2.compute_keypair.name
   region    = "RegionOne"
 
   user_data = data.cloudinit_config.init_docker_controller.rendered
+
   security_groups = ["default",
     openstack_networking_secgroup_v2.cogstack_apps_security_group.name
   ]
 
   network {
-    uuid = data.openstack_networking_network_v2.external_4003.id
+    uuid = local.network_id
   }
 
   block_device {
@@ -23,9 +24,17 @@ resource "openstack_compute_instance_v2" "kubernetes_server" {
     delete_on_termination = true
   }
 
+  lifecycle {
+    ignore_changes = [user_data]
+  }
+}
+
+resource "null_resource" "kubernetes_server_provisioner" {
+  depends_on = [openstack_compute_instance_v2.kubernetes_server, openstack_networking_floatingip_associate_v2.kubernetes_server_fip]
+
   connection {
     user        = "ubuntu"
-    host        = self.access_ip_v4
+    host        = local.controller_host_instance.ip_address
     private_key = file(local.ssh_keys.private_key_file)
   }
 
@@ -37,34 +46,55 @@ resource "openstack_compute_instance_v2" "kubernetes_server" {
 }
 
 resource "openstack_compute_instance_v2" "kubernetes_nodes" {
-  depends_on = [openstack_compute_instance_v2.kubernetes_server]
-  for_each   = { for vm in var.host_instances : vm.name => vm if !vm.is_controller }
-  name       = "${local.random_prefix}-${each.value.name}"
-  flavor_id  = data.openstack_compute_flavor_v2.available_compute_flavors[each.value.flavour].id
-  key_pair   = openstack_compute_keypair_v2.compute_keypair.name
-  region     = "RegionOne"
+  depends_on = [
+    openstack_compute_instance_v2.kubernetes_server
+  ]
+
+  for_each = { for vm in var.host_instances : vm.name => vm if !vm.is_controller }
+
+  name      = local.prefix != "" ? "${local.prefix}-${each.value.name}" : each.value.name
+  flavor_id = data.openstack_compute_flavor_v2.available_compute_flavors[each.value.flavour].id
+  key_pair  = openstack_compute_keypair_v2.compute_keypair.name
+  region    = "RegionOne"
 
-  user_data = data.cloudinit_config.init_docker.rendered
+  user_data = data.cloudinit_config.init_docker[each.key].rendered
   security_groups = ["default",
     openstack_networking_secgroup_v2.cogstack_apps_security_group.name
   ]
 
   network {
-    uuid = data.openstack_networking_network_v2.external_4003.id
+    uuid = local.network_id
   }
 
   block_device {
-    uuid                  = each.value.image_uuid == null ? data.openstack_images_image_v2.ubuntu.id : each.value.image_uuid 
+    uuid                  = each.value.image_uuid == null ? data.openstack_images_image_v2.ubuntu.id : each.value.image_uuid
     source_type           = "image"
     volume_size           = each.value.volume_size
     boot_index            = 0
     destination_type      = "volume"
     delete_on_termination = true
   }
 
+  lifecycle {
+    ignore_changes = [user_data]
+  }
+}
+
+locals {
+  is_default_network    = var.network != null && var.network == { name = "external_4003" }
+  nodes_with_exernal_ip = { for node in local.created_nodes : node.name => node if local.is_default_network || node.use_floating_ip }
+
+}
+resource "null_resource" "kubernetes_nodes_provisioner" {
+  # Provisioner is only used to check for node readiness. Skip for any nodes that are not in the external network,
+  # TODO: Filter this provisioner to only run on nodes that have a floating IP if the network is not default
+  for_each = local.nodes_with_exernal_ip
+
+  depends_on = [openstack_compute_instance_v2.kubernetes_nodes, openstack_networking_floatingip_associate_v2.kubernetes_nodes_fip]
+
   connection {
     user        = "ubuntu"
-    host        = self.access_ip_v4
+    host        = each.value.ip_address
     private_key = file(local.ssh_keys.private_key_file)
   }
 
@@ -75,8 +105,6 @@ resource "openstack_compute_instance_v2" "kubernetes_nodes" {
   }
 }
 
-
-
 # TODO: Read content from files and put into cloud-init config
 # data "local_file" "install_docker_sh" {
 #   filename = "${path.module}/resources/install-docker.sh"
@@ -88,6 +116,8 @@ resource "openstack_compute_instance_v2" "kubernetes_nodes" {
 
 
 data "cloudinit_config" "init_docker" {
+  for_each = { for vm in var.host_instances : vm.name => vm if !vm.is_controller }
+
   depends_on = [openstack_compute_instance_v2.kubernetes_server]
   part {
     filename     = "cloud-init-k3s-agent.yaml"
@@ -96,6 +126,7 @@ data "cloudinit_config" "init_docker" {
       {
         TF_K3S_TOKEN             = random_password.k3s_token.result
         TF_K3S_SERVER_IP_ADDRESS = openstack_compute_instance_v2.kubernetes_server.access_ip_v4
+        TF_K3S_NODE_EXTERNAL_IP  = each.value.floating_ip != null && each.value.floating_ip.use_floating_ip ? each.value.floating_ip.address : ""
       }
     )
   }
@@ -111,7 +142,9 @@ data "cloudinit_config" "init_docker_controller" {
     content_type = "text/cloud-config"
     content = templatefile("${path.module}/cloud-init-k3s-server.yaml",
       {
-        TF_K3S_TOKEN = random_password.k3s_token.result
+        TF_K3S_TOKEN            = random_password.k3s_token.result
+        TF_K3S_TLS_SAN          = local.controller_host_has_floating_ip ? local.controller_host.floating_ip.address : ""
+        TF_K3S_NODE_EXTERNAL_IP = local.controller_host_has_floating_ip ? local.controller_host.floating_ip.address : ""
       }
     )
   }
@@ -123,16 +156,12 @@ data "openstack_compute_flavor_v2" "available_compute_flavors" {
 }
 
 
-data "openstack_networking_network_v2" "external_4003" {
-  name = "external_4003"
+data "openstack_networking_network_v2" "network" {
+  name = var.network != null && var.network.name != null ? var.network.name : "external_4003"
 }
 
 data "openstack_images_image_v2" "ubuntu" {
   name        = var.ubuntu_immage_name
   most_recent = true
 }
 
-data "openstack_networking_secgroup_v2" "er_https_from_lbs" {
-  name = "er_https_from_lbs"
-}
-

deployment/terraform/modules/openstack-kubernetes-infra/kubeconfig-extraction.tf

@@ -1,18 +1,18 @@
 resource "null_resource" "copy_kubeconfig" {
-  depends_on = [openstack_compute_instance_v2.kubernetes_server]
+  depends_on = [openstack_compute_instance_v2.kubernetes_server, null_resource.kubernetes_server_provisioner]
 
   provisioner "local-exec" {
     # Copy the kubeconfig file from the host to a local file using SCP.
     # Use ssh-keyscan to prevent interactive prompt on unknown host
     # Use sed to replace the localhost address in the KUBECONFIG file with the actual IP adddress of the created VM. 
     command = <<EOT
 mkdir -p ${path.root}/.build/ && \
-ssh-keyscan -H ${openstack_compute_instance_v2.kubernetes_server.access_ip_v4} >> ${path.root}/.build/.known_hosts_cogstack && \
+ssh-keyscan -H ${local.controller_host_instance.ip_address} >> ${path.root}/.build/.known_hosts_cogstack && \
 ssh -o UserKnownHostsFile=${path.root}/.build/.known_hosts_cogstack -o StrictHostKeyChecking=yes \
     -i ${local.ssh_keys.private_key_file} \
-    ubuntu@${openstack_compute_instance_v2.kubernetes_server.access_ip_v4} \
+    ubuntu@${local.controller_host_instance.ip_address} \
     "sudo cat /etc/rancher/k3s/k3s.yaml" > ${local.kubeconfig_file} && \
-sed -i "s/127.0.0.1/${openstack_compute_instance_v2.kubernetes_server.access_ip_v4}/" ${local.kubeconfig_file}
+sed -i "s/127.0.0.1/${local.controller_host_instance.ip_address}/" ${local.kubeconfig_file}
 EOT
   }
 }

deployment/terraform/modules/openstack-kubernetes-infra/networking.tf

@@ -13,7 +13,7 @@ locals {
 }
 
 resource "openstack_networking_secgroup_v2" "cogstack_apps_security_group" {
-  name        = "${local.random_prefix}-cogstack-services"
+  name        = local.prefix != "" ? "${local.prefix}-cogstack-services" : "cogstack-services"
   description = "Cogstack Apps and Services Group"
 }
 
@@ -29,3 +29,31 @@ resource "openstack_networking_secgroup_rule_v2" "cogstack_apps_port_rules" {
   security_group_id = openstack_networking_secgroup_v2.cogstack_apps_security_group.id
 }
 
+
+
+# Look up ports by device_id and network_id
+data "openstack_networking_port_v2" "server_port" {
+  count      = local.controller_host_has_floating_ip ? 1 : 0
+  device_id  = openstack_compute_instance_v2.kubernetes_server.id
+  network_id = openstack_compute_instance_v2.kubernetes_server.network[0].uuid
+}
+
+data "openstack_networking_port_v2" "nodes_port" {
+  for_each   = { for vm in var.host_instances : vm.name => vm if !vm.is_controller && vm.floating_ip != null && vm.floating_ip.use_floating_ip }
+  device_id  = openstack_compute_instance_v2.kubernetes_nodes[each.key].id
+  network_id = openstack_compute_instance_v2.kubernetes_nodes[each.key].network[0].uuid
+}
+
+# Associate floating IP with kubernetes server
+resource "openstack_networking_floatingip_associate_v2" "kubernetes_server_fip" {
+  count       = local.controller_host_has_floating_ip ? 1 : 0
+  floating_ip = local.controller_host.floating_ip.address
+  port_id     = data.openstack_networking_port_v2.server_port[0].id
+}
+
+# Associate floating IPs with kubernetes nodes
+resource "openstack_networking_floatingip_associate_v2" "kubernetes_nodes_fip" {
+  for_each    = { for vm in var.host_instances : vm.name => vm if !vm.is_controller && vm.floating_ip != null && vm.floating_ip.use_floating_ip }
+  floating_ip = each.value.floating_ip.address
+  port_id     = data.openstack_networking_port_v2.nodes_port[each.key].id
+}

deployment/terraform/modules/openstack-kubernetes-infra/outputs.tf

@@ -1,10 +1,6 @@
 
 output "created_hosts" {
-  value = merge({ for k, value in openstack_compute_instance_v2.kubernetes_nodes : k => {
-    ip_address  = value.access_ip_v4
-    unique_name = value.name
-    name        = k
-    } },
+  value = merge(local.created_nodes,
     {
       (local.controller_host.name) : local.controller_host_instance
   })
@@ -29,3 +25,8 @@ output "kubeconfig_file" {
   value       = abspath(local.kubeconfig_file)
   description = "Path to the generated KUBECONFIG file used to connect to kubernetes"
 }
+
+output "created_security_group" {
+  value       = openstack_networking_secgroup_v2.cogstack_apps_security_group
+  description = "Security group associated to the created hosts"
+}