From aa00c1d91ac9f4b6053b10aacd3fd1a6b7f50381 Mon Sep 17 00:00:00 2001 From: Fredrik Liv Date: Fri, 10 Sep 2021 22:55:21 +0200 Subject: [PATCH] Updated UpCloud terraform script to use private network and dynamic (#7779) additional disks --- contrib/terraform/upcloud/README.md | 48 +++--- .../terraform/upcloud/cluster-settings.tfvars | 40 ++++- contrib/terraform/upcloud/main.tf | 16 +- .../modules/kubernetes-cluster/main.tf | 142 +++++++++++++----- .../modules/kubernetes-cluster/output.tf | 10 +- .../modules/kubernetes-cluster/variables.tf | 24 +-- .../upcloud/sample-inventory/cluster.tfvars | 40 ++++- contrib/terraform/upcloud/variables.tf | 33 +++- 8 files changed, 264 insertions(+), 89 deletions(-) diff --git a/contrib/terraform/upcloud/README.md b/contrib/terraform/upcloud/README.md index dcce236fed1..6481453fc5c 100644 --- a/contrib/terraform/upcloud/README.md +++ b/contrib/terraform/upcloud/README.md @@ -8,27 +8,29 @@ The setup looks like following ```text Kubernetes cluster -+-----------------------+ -| +--------------+ | -| | +--------------+ | -| | | | | -| | | Master/etcd | | -| | | node(s) | | -| +-+ | | -| +--------------+ | -| ^ | -| | | -| v | -| +--------------+ | -| | +--------------+ | -| | | | | -| | | Worker | | -| | | node(s) | | -| +-+ | | -| +--------------+ | -+-----------------------+ ++--------------------------+ +| +--------------+ | +| | +--------------+ | +| --> | | | | +| | | Master/etcd | | +| | | node(s) | | +| +-+ | | +| +--------------+ | +| ^ | +| | | +| v | +| +--------------+ | +| | +--------------+ | +| --> | | | | +| | | Worker | | +| | | node(s) | | +| +-+ | | +| +--------------+ | ++--------------------------+ ``` +The nodes uses a private network for node to node communication and a public interface for all external communication. + ## Requirements * Terraform 0.13.0 or newer @@ -94,9 +96,10 @@ terraform destroy --var-file cluster-settings.tfvars \ ## Variables -* `hostname`: A valid domain name, e.g. example.com. The maximum length is 128 characters. +* `prefix`: Prefix to add to all resources, if set to "" don't set any prefix * `template_name`: The name or UUID of a base image -* `username`: a user to access the nodes +* `username`: a user to access the nodes, defaults to "ubuntu" +* `private_network_cidr`: CIDR to use for the private network, defaults to "172.16.0.0/24" * `ssh_public_keys`: List of public SSH keys to install on all machines * `zone`: The zone where to run the cluster * `machines`: Machines to provision. Key of this object will be used as the name of the machine @@ -104,3 +107,6 @@ terraform destroy --var-file cluster-settings.tfvars \ * `cpu`: number of cpu cores * `mem`: memory size in MB * `disk_size`: The size of the storage in GB + * `additional_disks`: Additional disks to attach to the node. + * `size`: The size of the additional disk in GB + * `tier`: The tier of disk to use (`maxiops` is the only one you can choose atm) diff --git a/contrib/terraform/upcloud/cluster-settings.tfvars b/contrib/terraform/upcloud/cluster-settings.tfvars index 08bf5dacf31..217f46accbe 100644 --- a/contrib/terraform/upcloud/cluster-settings.tfvars +++ b/contrib/terraform/upcloud/cluster-settings.tfvars @@ -1,12 +1,11 @@ - # See: https://developers.upcloud.com/1.3/5-zones/ zone = "fi-hel1" username = "ubuntu" -inventory_file = "inventory.ini" +# Prefix to use for all resources to separate them from other resources +prefix = "kubespray" -# A valid domain name, e.g. host.example.com. The maximum length is 128 characters. -hostname = "example.com" +inventory_file = "inventory.ini" # Set the operating system using UUID or exact name template_name = "Ubuntu Server 20.04 LTS (Focal Fossa)" @@ -17,7 +16,7 @@ ssh_public_keys = [ "ssh-rsa public key 2", ] -#check list of available plan https://developers.upcloud.com/1.3/7-plans/ +# check list of available plan https://developers.upcloud.com/1.3/7-plans/ machines = { "master-0" : { "node_type" : "master", @@ -27,6 +26,7 @@ machines = { "mem" : "4096" # The size of the storage in GB "disk_size" : 250 + "additional_disks" : {} }, "worker-0" : { "node_type" : "worker", @@ -36,6 +36,16 @@ machines = { "mem" : "4096" # The size of the storage in GB "disk_size" : 250 + "additional_disks" : { + # "some-disk-name-1": { + # "size": 100, + # "tier": "maxiops", + # }, + # "some-disk-name-2": { + # "size": 100, + # "tier": "maxiops", + # } + } }, "worker-1" : { "node_type" : "worker", @@ -45,6 +55,16 @@ machines = { "mem" : "4096" # The size of the storage in GB "disk_size" : 250 + "additional_disks" : { + # "some-disk-name-1": { + # "size": 100, + # "tier": "maxiops", + # }, + # "some-disk-name-2": { + # "size": 100, + # "tier": "maxiops", + # } + } }, "worker-2" : { "node_type" : "worker", @@ -54,5 +74,15 @@ machines = { "mem" : "4096" # The size of the storage in GB "disk_size" : 250 + "additional_disks" : { + # "some-disk-name-1": { + # "size": 100, + # "tier": "maxiops", + # }, + # "some-disk-name-2": { + # "size": 100, + # "tier": "maxiops", + # } + } } } diff --git a/contrib/terraform/upcloud/main.tf b/contrib/terraform/upcloud/main.tf index 8ddd46542e9..b38171cc124 100644 --- a/contrib/terraform/upcloud/main.tf +++ b/contrib/terraform/upcloud/main.tf @@ -11,12 +11,14 @@ provider "upcloud" { module "kubernetes" { source = "./modules/kubernetes-cluster" - zone = var.zone - hostname = var.hostname + prefix = var.prefix + zone = var.zone template_name = var.template_name username = var.username + private_network_cidr = var.private_network_cidr + machines = var.machines ssh_public_keys = var.ssh_public_keys @@ -30,13 +32,15 @@ data "template_file" "inventory" { template = file("${path.module}/templates/inventory.tpl") vars = { - connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s etcd_member_name=etcd%d", + connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d", keys(module.kubernetes.master_ip), - values(module.kubernetes.master_ip), + values(module.kubernetes.master_ip).*.public_ip, + values(module.kubernetes.master_ip).*.private_ip, range(1, length(module.kubernetes.master_ip) + 1))) - connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s", + connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s", keys(module.kubernetes.worker_ip), - values(module.kubernetes.worker_ip))) + values(module.kubernetes.worker_ip).*.public_ip, + values(module.kubernetes.worker_ip).*.private_ip)) list_master = join("\n", formatlist("%s", keys(module.kubernetes.master_ip))) list_worker = join("\n", formatlist("%s", diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf index 6a79af720f6..c72788c6975 100644 --- a/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf +++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf @@ -1,3 +1,41 @@ +locals { + # Create a list of all disks to create + disks = flatten([ + for node_name, machine in var.machines : [ + for disk_name, disk in machine.additional_disks : { + disk = disk + disk_name = disk_name + node_name = node_name + } + ] + ]) + + # If prefix is set, all resources will be prefixed with "${var.prefix}-" + # Else don't prefix with anything + resource-prefix = "%{ if var.prefix != ""}${var.prefix}-%{ endif }" +} + +resource "upcloud_network" "private" { + name = "${local.resource-prefix}k8s-network" + zone = var.zone + + ip_network { + address = var.private_network_cidr + dhcp = true + family = "IPv4" + } +} + +resource "upcloud_storage" "additional_disks" { + for_each = { + for disk in local.disks: "${disk.node_name}_${disk.disk_name}" => disk.disk + } + + size = each.value.size + tier = each.value.tier + title = "${local.resource-prefix}${each.key}" + zone = var.zone +} resource "upcloud_server" "master" { for_each = { @@ -6,35 +44,48 @@ resource "upcloud_server" "master" { if machine.node_type == "master" } - hostname = "${each.key}.${var.hostname}" - cpu = each.value.cpu - mem = each.value.mem - zone = var.zone + hostname = "${local.resource-prefix}${each.key}" + cpu = each.value.cpu + mem = each.value.mem + zone = var.zone template { storage = var.template_name - size = each.value.disk_size + size = each.value.disk_size + } + + # Public network interface + network_interface { + type = "public" } - # Network interfaces - network_interface { - type = "public" - } + # Private network interface + network_interface { + type = "private" + network = upcloud_network.private.id + } - network_interface { - type = "utility" - } - # Include at least one public SSH key - login { - user = var.username - keys = var.ssh_public_keys - create_password = false + dynamic "storage_devices" { + for_each = { + for disk_key_name, disk in upcloud_storage.additional_disks : + disk_key_name => disk + # Only add the disk if it matches the node name in the start of its name + if length(regexall("^${each.key}_.+", disk_key_name)) > 0 + } - } + content { + storage = storage_devices.value.id + } + } + # Include at least one public SSH key + login { + user = var.username + keys = var.ssh_public_keys + create_password = false + } } - resource "upcloud_server" "worker" { for_each = { for name, machine in var.machines : @@ -42,25 +93,44 @@ resource "upcloud_server" "worker" { if machine.node_type == "worker" } - hostname = "${each.key}.${var.hostname}" - cpu = each.value.cpu - mem = each.value.mem - zone = var.zone + hostname = "${local.resource-prefix}${each.key}" + cpu = each.value.cpu + mem = each.value.mem + zone = var.zone template { - storage = var.template_name - size = each.value.disk_size + storage = var.template_name + size = each.value.disk_size + } + + # Public network interface + network_interface { + type = "public" } - # Network interfaces - network_interface { - type = "public" - } - - # Include at least one public SSH key - login { - user = var.username - keys = var.ssh_public_keys - create_password = false - } + # Private network interface + network_interface { + type = "private" + network = upcloud_network.private.id + } + + dynamic "storage_devices" { + for_each = { + for disk_key_name, disk in upcloud_storage.additional_disks : + disk_key_name => disk + # Only add the disk if it matches the node name in the start of its name + if length(regexall("^${each.key}_.+", disk_key_name)) > 0 + } + + content { + storage = storage_devices.value.id + } + } + + # Include at least one public SSH key + login { + user = var.username + keys = var.ssh_public_keys + create_password = false + } } diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf index 2661ac0136c..7343a80bb77 100644 --- a/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf +++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf @@ -2,13 +2,19 @@ output "master_ip" { value = { for instance in upcloud_server.master : - instance.hostname => instance.network_interface[0].ip_address + instance.hostname => { + "public_ip": instance.network_interface[0].ip_address + "private_ip": instance.network_interface[1].ip_address + } } } output "worker_ip" { value = { for instance in upcloud_server.worker : - instance.hostname => instance.network_interface[0].ip_address + instance.hostname => { + "public_ip": instance.network_interface[0].ip_address + "private_ip": instance.network_interface[1].ip_address + } } } diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf index 5b130ad1078..55abc509b77 100644 --- a/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf +++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf @@ -1,22 +1,28 @@ -variable "zone" { +variable "prefix" { type = string } -variable "hostname"{ - default ="example.com" +variable "zone" { + type = string } -variable "template_name"{} +variable "template_name" {} + +variable "username" {} -variable "username"{} +variable "private_network_cidr" {} variable "machines" { description = "Cluster machines" type = map(object({ - node_type = string - cpu = string - mem = string - disk_size = number + node_type = string + cpu = string + mem = string + disk_size = number + additional_disks = map(object({ + size = number + tier = string + })) })) } diff --git a/contrib/terraform/upcloud/sample-inventory/cluster.tfvars b/contrib/terraform/upcloud/sample-inventory/cluster.tfvars index 0a324045ec3..1400ed3e4cd 100644 --- a/contrib/terraform/upcloud/sample-inventory/cluster.tfvars +++ b/contrib/terraform/upcloud/sample-inventory/cluster.tfvars @@ -2,20 +2,21 @@ zone = "fi-hel1" username = "ubuntu" -inventory_file = "inventory.ini" +# Prefix to use for all resources to separate them from other resources +prefix = "kubespray" -# A valid domain name, e.g. host.example.com. The maximum length is 128 characters. -hostname = "example.com" +inventory_file = "inventory.ini" # Set the operating system using UUID or exact name template_name = "Ubuntu Server 20.04 LTS (Focal Fossa)" + ssh_public_keys = [ # Put your public SSH key here "ssh-rsa I-did-not-read-the-docs", "ssh-rsa I-did-not-read-the-docs 2", ] -check list of available plan https://developers.upcloud.com/1.3/7-plans/ +# check list of available plan https://developers.upcloud.com/1.3/7-plans/ machines = { "master-0" : { "node_type" : "master", @@ -25,6 +26,7 @@ machines = { "mem" : "4096" # The size of the storage in GB "disk_size" : 250 + "additional_disks": {} }, "worker-0" : { "node_type" : "worker", @@ -34,6 +36,16 @@ machines = { "mem" : "4096" # The size of the storage in GB "disk_size" : 250 + "additional_disks": { + # "some-disk-name-1": { + # "size": 100, + # "tier": "maxiops", + # }, + # "some-disk-name-2": { + # "size": 100, + # "tier": "maxiops", + # } + } }, "worker-1" : { "node_type" : "worker", @@ -43,6 +55,16 @@ machines = { "mem" : "4096" # The size of the storage in GB "disk_size" : 250 + "additional_disks": { + # "some-disk-name-1": { + # "size": 100, + # "tier": "maxiops", + # }, + # "some-disk-name-2": { + # "size": 100, + # "tier": "maxiops", + # } + } }, "worker-2" : { "node_type" : "worker", @@ -52,5 +74,15 @@ machines = { "mem" : "4096" # The size of the storage in GB "disk_size" : 250 + "additional_disks": { + # "some-disk-name-1": { + # "size": 100, + # "tier": "maxiops", + # }, + # "some-disk-name-2": { + # "size": 100, + # "tier": "maxiops", + # } + } } } diff --git a/contrib/terraform/upcloud/variables.tf b/contrib/terraform/upcloud/variables.tf index 60941d57266..95c90f2f10c 100644 --- a/contrib/terraform/upcloud/variables.tf +++ b/contrib/terraform/upcloud/variables.tf @@ -1,23 +1,40 @@ +variable "prefix" { + type = string + default = "kubespray" + + description = "Prefix that is used to distinguish these resources from others" +} variable "zone" { description = "The zone where to run the cluster" } -variable "hostname" { - default = "example.com" +variable "template_name" { + description = "Block describing the preconfigured operating system" } -variable "template_name" {} +variable "username" { + description = "The username to use for the nodes" + default = "ubuntu" +} -variable "username" {} +variable "private_network_cidr" { + description = "CIDR to use for the private network" + default = "172.16.0.0/24" +} variable "machines" { description = "Cluster machines" + type = map(object({ node_type = string cpu = string mem = string disk_size = number + additional_disks = map(object({ + size = number + tier = string + })) })) } @@ -30,6 +47,10 @@ variable "inventory_file" { description = "Where to store the generated inventory file" } -variable "UPCLOUD_USERNAME" {} +variable "UPCLOUD_USERNAME" { + description = "UpCloud username with API access" +} -variable "UPCLOUD_PASSWORD" {} +variable "UPCLOUD_PASSWORD" { + description = "Password for UpCloud API user" +}