diff --git a/deployment/terraform/examples/openstack-kubernetes/k3s-cluster/main.tf b/deployment/terraform/examples/openstack-kubernetes/k3s-cluster/main.tf index 437d004..7e4e4bd 100644 --- a/deployment/terraform/examples/openstack-kubernetes/k3s-cluster/main.tf +++ b/deployment/terraform/examples/openstack-kubernetes/k3s-cluster/main.tf @@ -1,7 +1,14 @@ module "openstack_cogstack_infra" { source = "../../../modules/openstack-kubernetes-infra" host_instances = [ - { name = "cogstack-k3s", is_controller = true }, + { + name = "cogstack-k3s", + is_controller = true, + # floating_ip = { + # use_floating_ip = true, + # address = "10.10.10.10" + # } + }, { name = "cogstack-k3s-node-2" flavour = "2cpu4ram" @@ -11,4 +18,9 @@ module "openstack_cogstack_infra" { ] allowed_ingress_ips_cidr = var.allowed_ingress_ips_cidr ubuntu_immage_name = var.ubuntu_immage_name + # generate_random_name_prefix = false + # prefix = "dev" + # network = { + # network_id = "some-id" + # } } diff --git a/deployment/terraform/modules/openstack-cogstack-infra/compute.tf b/deployment/terraform/modules/openstack-cogstack-infra/compute.tf index 0ab8c89..7764a68 100644 --- a/deployment/terraform/modules/openstack-cogstack-infra/compute.tf +++ b/deployment/terraform/modules/openstack-cogstack-infra/compute.tf @@ -125,6 +125,3 @@ data "openstack_images_image_v2" "ubuntu" { most_recent = true } -data "openstack_networking_secgroup_v2" "er_https_from_lbs" { - name = "er_https_from_lbs" -} diff --git a/deployment/terraform/modules/openstack-kubernetes-infra/README.md b/deployment/terraform/modules/openstack-kubernetes-infra/README.md index 466a0e7..b21981b 100644 --- a/deployment/terraform/modules/openstack-kubernetes-infra/README.md +++ b/deployment/terraform/modules/openstack-kubernetes-infra/README.md @@ -35,4 +35,54 @@ module "openstack_kubernetes_cluster" { } ``` +## Existing Network and Floating IPs +You can use this module with a custom OpenStack network by specifying the `network` variable. By default, it will use the network named `"external_4003"`, but you can override this with your own network name or network ID. + +Using a custom OpenStack network with your own subnet allows you to improve security by keeping most nodes private. This ensures worker nodes and other internal resources are not directly exposed, reducing security risks. Only the controller node needs a floating IP to be accessible in order to use the k8s api server (eg for kubectl to work from outside the network). Using floating ips will also have the advantage of being stable, so you can destroy/create VMs without having to update anywhere the IP address is referenced by reassigning the floating ip. + +To use this configuration, you will probably need to assign a floating IP to the controller node so that it is accessible. You can configure floating IP assignment per node using the `floating_ip` block within each entry in the `host_instances` variable + +Please see this example for using thism module with a custom network and floating IPs. + +```hcl +host_instances = [ + { + name = "controller" + is_controller = true + floating_ip = { + use_floating_ip = true + address = "203.0.113.10" # Address of an existing floating_ip in openstack + } + }, + { + name = "worker" + # Optionally also assign a floating IP here, or leave blank to keep it internal to the network + } + network = { + network_id = openstack_networking_network_v2.example_network.id + } +] + + +resource "openstack_networking_network_v2" "example_network" { + name = "dev-example-network" + admin_state_up = "true" +} + +resource "openstack_networking_subnet_v2" "example_subnet"{ + name = "dev-example-subnet" + network_id = openstack_networking_network_v2.example_network.id + cidr = "192.168.0.0/24" + ip_version = 4 +} + +resource "openstack_networking_router_v2" "example_router" { + name = "test-router" + admin_state_up = true + external_network_id = data.openstack_networking_network_v2.external_4003.id +} +``` + +When using a non-default network, ensure the controller host has a floating IP so Terraform can access it for provisioning. + diff --git a/deployment/terraform/modules/openstack-kubernetes-infra/cloud-init-k3s-agent.yaml b/deployment/terraform/modules/openstack-kubernetes-infra/cloud-init-k3s-agent.yaml index 36aa265..df2b8a3 100644 --- a/deployment/terraform/modules/openstack-kubernetes-infra/cloud-init-k3s-agent.yaml +++ b/deployment/terraform/modules/openstack-kubernetes-infra/cloud-init-k3s-agent.yaml @@ -37,7 +37,13 @@ runcmd: # Run K3s - echo "Installing K3S" - - curl -sfL https://get.k3s.io | K3S_URL=https://${TF_K3S_SERVER_IP_ADDRESS}:6443 K3S_TOKEN="${TF_K3S_TOKEN}" sh - + - | + INSTALL_K3S_EXEC="" + TF_K3S_NODE_EXTERNAL_IP=${TF_K3S_NODE_EXTERNAL_IP} + if [ -n "$TF_K3S_NODE_EXTERNAL_IP" ]; then + INSTALL_K3S_EXEC="--node-external-ip $TF_K3S_NODE_EXTERNAL_IP" + fi + curl -sfL https://get.k3s.io | K3S_URL=https://${TF_K3S_SERVER_IP_ADDRESS}:6443 K3S_TOKEN="${TF_K3S_TOKEN}" INSTALL_K3S_EXEC="$INSTALL_K3S_EXEC" sh - - echo "Completed Installing K3S" # - sudo chmod 644 /etc/rancher/k3s/k3s.yaml diff --git a/deployment/terraform/modules/openstack-kubernetes-infra/cloud-init-k3s-server.yaml b/deployment/terraform/modules/openstack-kubernetes-infra/cloud-init-k3s-server.yaml index a7c379a..8d87e26 100644 --- a/deployment/terraform/modules/openstack-kubernetes-infra/cloud-init-k3s-server.yaml +++ b/deployment/terraform/modules/openstack-kubernetes-infra/cloud-init-k3s-server.yaml @@ -37,7 +37,17 @@ runcmd: # Run K3s - echo "Installing K3S" - - sudo curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server" K3S_TOKEN="${TF_K3S_TOKEN}" sh - + - | + TF_K3S_TLS_SAN=${TF_K3S_TLS_SAN} + TF_K3S_NODE_EXTERNAL_IP=${TF_K3S_NODE_EXTERNAL_IP} + INSTALL_K3S_EXEC="server" + if [ -n "$TF_K3S_TLS_SAN" ]; then + INSTALL_K3S_EXEC="$INSTALL_K3S_EXEC --tls-san $TF_K3S_TLS_SAN" + fi + if [ -n "$TF_K3S_NODE_EXTERNAL_IP" ]; then + INSTALL_K3S_EXEC="$INSTALL_K3S_EXEC --node-external-ip $TF_K3S_NODE_EXTERNAL_IP" + fi + sudo curl -sfL https://get.k3s.io | K3S_TOKEN="${TF_K3S_TOKEN}" INSTALL_K3S_EXEC="$INSTALL_K3S_EXEC" sh - - echo "Completed Installing K3S" - sudo chmod 644 /etc/rancher/k3s/k3s.yaml diff --git a/deployment/terraform/modules/openstack-kubernetes-infra/compute-keypair.tf b/deployment/terraform/modules/openstack-kubernetes-infra/compute-keypair.tf index 7089791..7d18a30 100644 --- a/deployment/terraform/modules/openstack-kubernetes-infra/compute-keypair.tf +++ b/deployment/terraform/modules/openstack-kubernetes-infra/compute-keypair.tf @@ -14,7 +14,7 @@ locals { } resource "openstack_compute_keypair_v2" "compute_keypair" { - name = "${local.random_prefix}-cogstack_keypair" + name = local.prefix != "" ? "${local.prefix}-cogstack_keypair" : "cogstack_keypair" public_key = local.is_using_existing_ssh_keypair ? file(var.ssh_key_pair.public_key_file) : null } @@ -30,4 +30,4 @@ resource "local_file" "public_key" { content = openstack_compute_keypair_v2.compute_keypair.public_key filename = "${local.output_file_directory}/${openstack_compute_keypair_v2.compute_keypair.name}-rsa.pub" file_permission = "0600" -} \ No newline at end of file +} diff --git a/deployment/terraform/modules/openstack-kubernetes-infra/compute.tf b/deployment/terraform/modules/openstack-kubernetes-infra/compute.tf index b5a5551..869e888 100644 --- a/deployment/terraform/modules/openstack-kubernetes-infra/compute.tf +++ b/deployment/terraform/modules/openstack-kubernetes-infra/compute.tf @@ -1,17 +1,18 @@ resource "openstack_compute_instance_v2" "kubernetes_server" { - name = "${local.random_prefix}-${local.controller_host.name}" + name = local.prefix != "" ? "${local.prefix}-${local.controller_host.name}" : local.controller_host.name flavor_id = data.openstack_compute_flavor_v2.available_compute_flavors[local.controller_host.flavour].id key_pair = openstack_compute_keypair_v2.compute_keypair.name region = "RegionOne" user_data = data.cloudinit_config.init_docker_controller.rendered + security_groups = ["default", openstack_networking_secgroup_v2.cogstack_apps_security_group.name ] network { - uuid = data.openstack_networking_network_v2.external_4003.id + uuid = local.network_id } block_device { @@ -23,9 +24,17 @@ resource "openstack_compute_instance_v2" "kubernetes_server" { delete_on_termination = true } + lifecycle { + ignore_changes = [user_data] + } +} + +resource "null_resource" "kubernetes_server_provisioner" { + depends_on = [openstack_compute_instance_v2.kubernetes_server, openstack_networking_floatingip_associate_v2.kubernetes_server_fip] + connection { user = "ubuntu" - host = self.access_ip_v4 + host = local.controller_host_instance.ip_address private_key = file(local.ssh_keys.private_key_file) } @@ -37,24 +46,28 @@ resource "openstack_compute_instance_v2" "kubernetes_server" { } resource "openstack_compute_instance_v2" "kubernetes_nodes" { - depends_on = [openstack_compute_instance_v2.kubernetes_server] - for_each = { for vm in var.host_instances : vm.name => vm if !vm.is_controller } - name = "${local.random_prefix}-${each.value.name}" - flavor_id = data.openstack_compute_flavor_v2.available_compute_flavors[each.value.flavour].id - key_pair = openstack_compute_keypair_v2.compute_keypair.name - region = "RegionOne" + depends_on = [ + openstack_compute_instance_v2.kubernetes_server + ] + + for_each = { for vm in var.host_instances : vm.name => vm if !vm.is_controller } + + name = local.prefix != "" ? "${local.prefix}-${each.value.name}" : each.value.name + flavor_id = data.openstack_compute_flavor_v2.available_compute_flavors[each.value.flavour].id + key_pair = openstack_compute_keypair_v2.compute_keypair.name + region = "RegionOne" - user_data = data.cloudinit_config.init_docker.rendered + user_data = data.cloudinit_config.init_docker[each.key].rendered security_groups = ["default", openstack_networking_secgroup_v2.cogstack_apps_security_group.name ] network { - uuid = data.openstack_networking_network_v2.external_4003.id + uuid = local.network_id } block_device { - uuid = each.value.image_uuid == null ? data.openstack_images_image_v2.ubuntu.id : each.value.image_uuid + uuid = each.value.image_uuid == null ? data.openstack_images_image_v2.ubuntu.id : each.value.image_uuid source_type = "image" volume_size = each.value.volume_size boot_index = 0 @@ -62,9 +75,26 @@ resource "openstack_compute_instance_v2" "kubernetes_nodes" { delete_on_termination = true } + lifecycle { + ignore_changes = [user_data] + } +} + +locals { + is_default_network = var.network != null && var.network == { name = "external_4003" } + nodes_with_exernal_ip = { for node in local.created_nodes : node.name => node if local.is_default_network || node.use_floating_ip } + +} +resource "null_resource" "kubernetes_nodes_provisioner" { + # Provisioner is only used to check for node readiness. Skip for any nodes that are not in the external network, + # TODO: Filter this provisioner to only run on nodes that have a floating IP if the network is not default + for_each = local.nodes_with_exernal_ip + + depends_on = [openstack_compute_instance_v2.kubernetes_nodes, openstack_networking_floatingip_associate_v2.kubernetes_nodes_fip] + connection { user = "ubuntu" - host = self.access_ip_v4 + host = each.value.ip_address private_key = file(local.ssh_keys.private_key_file) } @@ -75,8 +105,6 @@ resource "openstack_compute_instance_v2" "kubernetes_nodes" { } } - - # TODO: Read content from files and put into cloud-init config # data "local_file" "install_docker_sh" { # filename = "${path.module}/resources/install-docker.sh" @@ -88,6 +116,8 @@ resource "openstack_compute_instance_v2" "kubernetes_nodes" { data "cloudinit_config" "init_docker" { + for_each = { for vm in var.host_instances : vm.name => vm if !vm.is_controller } + depends_on = [openstack_compute_instance_v2.kubernetes_server] part { filename = "cloud-init-k3s-agent.yaml" @@ -96,6 +126,7 @@ data "cloudinit_config" "init_docker" { { TF_K3S_TOKEN = random_password.k3s_token.result TF_K3S_SERVER_IP_ADDRESS = openstack_compute_instance_v2.kubernetes_server.access_ip_v4 + TF_K3S_NODE_EXTERNAL_IP = each.value.floating_ip != null && each.value.floating_ip.use_floating_ip ? each.value.floating_ip.address : "" } ) } @@ -111,7 +142,9 @@ data "cloudinit_config" "init_docker_controller" { content_type = "text/cloud-config" content = templatefile("${path.module}/cloud-init-k3s-server.yaml", { - TF_K3S_TOKEN = random_password.k3s_token.result + TF_K3S_TOKEN = random_password.k3s_token.result + TF_K3S_TLS_SAN = local.controller_host_has_floating_ip ? local.controller_host.floating_ip.address : "" + TF_K3S_NODE_EXTERNAL_IP = local.controller_host_has_floating_ip ? local.controller_host.floating_ip.address : "" } ) } @@ -123,8 +156,8 @@ data "openstack_compute_flavor_v2" "available_compute_flavors" { } -data "openstack_networking_network_v2" "external_4003" { - name = "external_4003" +data "openstack_networking_network_v2" "network" { + name = var.network != null && var.network.name != null ? var.network.name : "external_4003" } data "openstack_images_image_v2" "ubuntu" { @@ -132,7 +165,3 @@ data "openstack_images_image_v2" "ubuntu" { most_recent = true } -data "openstack_networking_secgroup_v2" "er_https_from_lbs" { - name = "er_https_from_lbs" -} - diff --git a/deployment/terraform/modules/openstack-kubernetes-infra/kubeconfig-extraction.tf b/deployment/terraform/modules/openstack-kubernetes-infra/kubeconfig-extraction.tf index 7a7d3ef..7e3fafe 100644 --- a/deployment/terraform/modules/openstack-kubernetes-infra/kubeconfig-extraction.tf +++ b/deployment/terraform/modules/openstack-kubernetes-infra/kubeconfig-extraction.tf @@ -1,5 +1,5 @@ resource "null_resource" "copy_kubeconfig" { - depends_on = [openstack_compute_instance_v2.kubernetes_server] + depends_on = [openstack_compute_instance_v2.kubernetes_server, null_resource.kubernetes_server_provisioner] provisioner "local-exec" { # Copy the kubeconfig file from the host to a local file using SCP. @@ -7,12 +7,12 @@ resource "null_resource" "copy_kubeconfig" { # Use sed to replace the localhost address in the KUBECONFIG file with the actual IP adddress of the created VM. command = <> ${path.root}/.build/.known_hosts_cogstack && \ +ssh-keyscan -H ${local.controller_host_instance.ip_address} >> ${path.root}/.build/.known_hosts_cogstack && \ ssh -o UserKnownHostsFile=${path.root}/.build/.known_hosts_cogstack -o StrictHostKeyChecking=yes \ -i ${local.ssh_keys.private_key_file} \ - ubuntu@${openstack_compute_instance_v2.kubernetes_server.access_ip_v4} \ + ubuntu@${local.controller_host_instance.ip_address} \ "sudo cat /etc/rancher/k3s/k3s.yaml" > ${local.kubeconfig_file} && \ -sed -i "s/127.0.0.1/${openstack_compute_instance_v2.kubernetes_server.access_ip_v4}/" ${local.kubeconfig_file} +sed -i "s/127.0.0.1/${local.controller_host_instance.ip_address}/" ${local.kubeconfig_file} EOT } } diff --git a/deployment/terraform/modules/openstack-kubernetes-infra/networking.tf b/deployment/terraform/modules/openstack-kubernetes-infra/networking.tf index 2b1c073..3cb08d0 100644 --- a/deployment/terraform/modules/openstack-kubernetes-infra/networking.tf +++ b/deployment/terraform/modules/openstack-kubernetes-infra/networking.tf @@ -13,7 +13,7 @@ locals { } resource "openstack_networking_secgroup_v2" "cogstack_apps_security_group" { - name = "${local.random_prefix}-cogstack-services" + name = local.prefix != "" ? "${local.prefix}-cogstack-services" : "cogstack-services" description = "Cogstack Apps and Services Group" } @@ -29,3 +29,31 @@ resource "openstack_networking_secgroup_rule_v2" "cogstack_apps_port_rules" { security_group_id = openstack_networking_secgroup_v2.cogstack_apps_security_group.id } + + +# Look up ports by device_id and network_id +data "openstack_networking_port_v2" "server_port" { + count = local.controller_host_has_floating_ip ? 1 : 0 + device_id = openstack_compute_instance_v2.kubernetes_server.id + network_id = openstack_compute_instance_v2.kubernetes_server.network[0].uuid +} + +data "openstack_networking_port_v2" "nodes_port" { + for_each = { for vm in var.host_instances : vm.name => vm if !vm.is_controller && vm.floating_ip != null && vm.floating_ip.use_floating_ip } + device_id = openstack_compute_instance_v2.kubernetes_nodes[each.key].id + network_id = openstack_compute_instance_v2.kubernetes_nodes[each.key].network[0].uuid +} + +# Associate floating IP with kubernetes server +resource "openstack_networking_floatingip_associate_v2" "kubernetes_server_fip" { + count = local.controller_host_has_floating_ip ? 1 : 0 + floating_ip = local.controller_host.floating_ip.address + port_id = data.openstack_networking_port_v2.server_port[0].id +} + +# Associate floating IPs with kubernetes nodes +resource "openstack_networking_floatingip_associate_v2" "kubernetes_nodes_fip" { + for_each = { for vm in var.host_instances : vm.name => vm if !vm.is_controller && vm.floating_ip != null && vm.floating_ip.use_floating_ip } + floating_ip = each.value.floating_ip.address + port_id = data.openstack_networking_port_v2.nodes_port[each.key].id +} diff --git a/deployment/terraform/modules/openstack-kubernetes-infra/outputs.tf b/deployment/terraform/modules/openstack-kubernetes-infra/outputs.tf index 6289ed1..e794092 100644 --- a/deployment/terraform/modules/openstack-kubernetes-infra/outputs.tf +++ b/deployment/terraform/modules/openstack-kubernetes-infra/outputs.tf @@ -1,10 +1,6 @@ output "created_hosts" { - value = merge({ for k, value in openstack_compute_instance_v2.kubernetes_nodes : k => { - ip_address = value.access_ip_v4 - unique_name = value.name - name = k - } }, + value = merge(local.created_nodes, { (local.controller_host.name) : local.controller_host_instance }) @@ -29,3 +25,8 @@ output "kubeconfig_file" { value = abspath(local.kubeconfig_file) description = "Path to the generated KUBECONFIG file used to connect to kubernetes" } + +output "created_security_group" { + value = openstack_networking_secgroup_v2.cogstack_apps_security_group + description = "Security group associated to the created hosts" +} \ No newline at end of file diff --git a/deployment/terraform/modules/openstack-kubernetes-infra/shared-locals.tf b/deployment/terraform/modules/openstack-kubernetes-infra/shared-locals.tf index e2d95c2..cff1b88 100644 --- a/deployment/terraform/modules/openstack-kubernetes-infra/shared-locals.tf +++ b/deployment/terraform/modules/openstack-kubernetes-infra/shared-locals.tf @@ -1,17 +1,35 @@ locals { random_prefix = random_id.server.b64_url + prefix = var.prefix != null ? var.prefix : (var.generate_random_name_prefix ? local.random_prefix : "") + network_id = var.network != null && var.network.network_id != null ? var.network.network_id : data.openstack_networking_network_v2.network.id } locals { - controller_host = one([for host in var.host_instances : host if host.is_controller]) - created_controller_host = openstack_compute_instance_v2.kubernetes_server + controller_host = one([for host in var.host_instances : host if host.is_controller]) + controller_host_has_floating_ip = local.controller_host.floating_ip != null && local.controller_host.floating_ip.use_floating_ip + created_controller_host = openstack_compute_instance_v2.kubernetes_server controller_host_instance = { - name = local.controller_host.name - ip_address = local.created_controller_host.access_ip_v4 - unique_name = local.created_controller_host.name + name = local.controller_host.name + ip_address = local.controller_host_has_floating_ip ? local.controller_host.floating_ip.address : openstack_compute_instance_v2.kubernetes_server.access_ip_v4 + unique_name = local.created_controller_host.name + use_floating_ip = local.controller_host_has_floating_ip + internal_ip_address = openstack_compute_instance_v2.kubernetes_server.access_ip_v4 } + + created_nodes = { + for node in var.host_instances : + node.name => { + ip_address = node.floating_ip != null && node.floating_ip.use_floating_ip ? node.floating_ip.address : openstack_compute_instance_v2.kubernetes_nodes[node.name].access_ip_v4 + unique_name = openstack_compute_instance_v2.kubernetes_nodes[node.name].name + name = node.name + use_floating_ip = node.floating_ip != null && node.floating_ip.use_floating_ip + internal_ip_address = openstack_compute_instance_v2.kubernetes_nodes[node.name].access_ip_v4 + } + if !node.is_controller + } + } locals { @@ -26,4 +44,4 @@ resource "random_id" "server" { } byte_length = 4 -} \ No newline at end of file +} diff --git a/deployment/terraform/modules/openstack-kubernetes-infra/variables.tf b/deployment/terraform/modules/openstack-kubernetes-infra/variables.tf index f89f65b..a548e91 100644 --- a/deployment/terraform/modules/openstack-kubernetes-infra/variables.tf +++ b/deployment/terraform/modules/openstack-kubernetes-infra/variables.tf @@ -17,13 +17,18 @@ is_controller = Must be true for exactly one host. This will run the k3s "server flavour = The openstack_compute_flavor_v2 for the host volume_size = Size in GB for the disk volume for the node image_uuid = (Optional) The Openstack image you want to run, to override the default in ubuntu_immage_name +floating_ip = (Optional) Floating IP configuration. Set use_floating_ip to true and provide address to associate a floating IP with this host EOT type = list(object({ name = string, flavour = optional(string, "2cpu4ram"), volume_size = optional(number, 20), is_controller = optional(bool, false), - image_uuid = optional(string, null) + image_uuid = optional(string, null), + floating_ip = optional(object({ + use_floating_ip = optional(bool, false) # Using a boolean to make it a plan time value. + address = optional(string) + }), null) })) default = [ @@ -62,4 +67,41 @@ variable "output_file_directory" { type = string default = null description = "Optional path to write output files to. If directory doesnt exist it will be created" -} \ No newline at end of file +} + +variable "generate_random_name_prefix" { + type = bool + default = true + description = "Whether to generate a random prefix for hostnames. If false, hostnames will use only the name from host_instances" +} + +variable "prefix" { + type = string + default = null + description = "Optional custom prefix for resource names. If provided will override generate_random_name_prefix" +} + +variable "network" { + type = object({ + name = optional(string, "external_4003") + network_id = optional(string) + }) + default = { name = "external_4003" } + description = "Network configuration. Either provide 'name' to lookup the network by name, or 'network_id' to use a network UUID directly. Defaults to name 'external_4003' if null" + validation { + condition = var.network == null || var.network.name != null || var.network.network_id != null + error_message = "Either network.name or network.network_id must be provided" + } +} + +check "controller_floating_ip_required_for_non_default_network" { + assert { + condition = (var.network == null + || (var.network.name == "external_4003" && var.network.network_id == null) + || (length([for host in var.host_instances : host if host.is_controller == true && host.floating_ip.use_floating_ip == true]) == 1)) + error_message = <<-EOT +When using a non-default network, the controller host should have a floating IP in order to be accessible by the terraform agent +EOT + } +} +