Support provisoning of Windows Server 2022 in ostests

This is an initial step to be able to test Windows clusters.

Signed-off-by: Tom Wieczorek <[email protected]>

hack/ostests/README.md

@@ -91,6 +91,7 @@ tofu apply
 * `ubuntu_2004`: Ubuntu 20.04 LTS
 * `ubuntu_2204`: Ubuntu 22.04 LTS
 * `ubuntu_2304`: Ubuntu 23.04
+* `windows_server_2022`: Windows Server 2022 (runs the control plane on Alpine 3.20)
 
 ### `arch`: Node architecture
 
@@ -125,6 +126,11 @@ tofu output -json | jq -r '
 
 This may be a fixed version number, "stable" or "latest".
 
+### `k0s_executable_path`: The k0s version to deploy
+
+Path to the k0s executable to use, or null if it should be downloaded. Note that
+for Windows, the `.exe` suffix is appended automatically.
+
 ### `k0s_network_provider`: Network provider
 
 * `kuberouter`

hack/ostests/modules/infra/main.tf

@@ -1,5 +1,7 @@
 resource "tls_private_key" "ssh" {
-  algorithm = "ED25519"
+  # For Windows instances, AWS enforces RSA, and disallows ed25519 ¯\_(ツ)_/¯
+  algorithm = "RSA"
+  rsa_bits  = 4096
 }
 
 resource "aws_key_pair" "ssh" {
@@ -8,31 +10,52 @@ resource "aws_key_pair" "ssh" {
 }
 
 locals {
-  default_node_config = {
+  default_node_config = merge ({
+    os_type = "linux"
+    volume  = { size = 20 }
+  }, {
     x86_64 = { instance_type = "t3a.small" }
     arm64  = { instance_type = "t4g.small" }
-  }[var.os.arch]
+  }[var.os.arch])
 
-  node_roles = {
+  node_role_templates = {
     controller = {
-      count         = var.controller_num_nodes
-      is_controller = true, is_worker = false,
-      node_config   = merge(local.default_node_config, var.os.node_configs.default, var.os.node_configs.controller)
+      data = {
+        count         = var.controller_num_nodes
+        is_controller = true, is_worker = false,
+      }
+      sources = [for s in [var.os.node_configs.controller, var.os.node_configs.default]: s if s != null]
     }
 
     "controller+worker" = {
-      count         = var.controller_worker_num_nodes
-      is_controller = true, is_worker = true,
-      node_config   = merge(local.default_node_config, var.os.node_configs.default, var.os.node_configs.worker, var.os.node_configs.controller_worker)
+      data = {
+        count         = var.controller_worker_num_nodes
+        is_controller = true, is_worker = true,
+      }
+      sources = [for s in [var.os.node_configs.controller_worker, var.os.node_configs.worker, var.os.node_configs.default]: s if s != null]
     }
 
     worker = {
-      count         = var.worker_num_nodes
-      is_controller = false, is_worker = true,
-      node_config   = merge(local.default_node_config, var.os.node_configs.default, var.os.node_configs.worker)
+      data = {
+        count         = var.worker_num_nodes
+        is_controller = false, is_worker = true,
+      }
+      sources = [for s in [var.os.node_configs.worker, var.os.node_configs.default]: s if s != null]
     }
   }
 
+  node_roles = { for role, tmpl in local.node_role_templates: role => merge(tmpl.data, {
+    node_config = {
+      ami_id        =  coalesce(tmpl.sources.*.ami_id...)
+      instance_type =  coalesce(concat(tmpl.sources, [local.default_node_config]).*.instance_type...)
+      os_type       =  coalesce(concat(tmpl.sources, [local.default_node_config]).*.os_type...)
+      volume        =  coalesce(concat(tmpl.sources, [local.default_node_config]).*.volume...)
+      user_data     =  try(coalesce(tmpl.sources.*.user_data...), null)
+      ready_script  =  try(coalesce(tmpl.sources.*.ready_script...), null)
+      connection    =  coalesce(tmpl.sources.*.connection...)
+    }
+  })}
+
   nodes = merge([for role, params in local.node_roles : {
     for idx in range(params.count) : "${role}-${idx + 1}" => {
       role          = role
@@ -68,19 +91,20 @@ resource "aws_instance" "nodes" {
 
   root_block_device {
     volume_type = "gp2"
-    volume_size = 20
+    volume_size = each.value.node_config.volume.size
   }
 }
 
 resource "terraform_data" "ready_scripts" {
   for_each = { for name, params in local.nodes : name => params if params.node_config.ready_script != null }
 
   connection {
-    type        = each.value.node_config.connection.type
-    user        = each.value.node_config.connection.username
-    private_key = tls_private_key.ssh.private_key_pem
-    host        = aws_instance.nodes[each.key].public_ip
-    agent       = false
+    type            = each.value.node_config.connection.type
+    user            = each.value.node_config.connection.username
+    target_platform = each.value.node_config.os_type == "windows" ? "windows" : "unix"
+    private_key     = tls_private_key.ssh.private_key_pem
+    host            = aws_instance.nodes[each.key].public_ip
+    agent           = false
   }
 
   provisioner "remote-exec" {
@@ -93,10 +117,11 @@ resource "terraform_data" "provisioned_nodes" {
 
   input = [for node in aws_instance.nodes : {
     name          = node.tags.Name,
-    ipv4          = node.public_ip,
+    os_type       = local.nodes[node.tags["ostests.k0sproject.io/node-name"]].node_config.os_type,
     role          = node.tags["k0sctl.k0sproject.io/host-role"]
     is_controller = local.nodes[node.tags["ostests.k0sproject.io/node-name"]].is_controller
     is_worker     = local.nodes[node.tags["ostests.k0sproject.io/node-name"]].is_worker
+    ipv4          = node.public_ip,
     connection    = local.nodes[node.tags["ostests.k0sproject.io/node-name"]].node_config.connection
   }]
 }

hack/ostests/modules/infra/variables.tf

@@ -13,7 +13,13 @@ variable "os" {
     arch = string
     node_configs = object({
       default = object({
-        ami_id = string
+        ami_id        = string
+        instance_type = optional(string)
+        os_type       = optional(string)
+
+        volume = optional(object({
+          size = number
+        }))
 
         user_data    = optional(string)
         ready_script = optional(string)
@@ -23,9 +29,60 @@ variable "os" {
           username = string
         })
       })
-      controller        = optional(map(any))
-      controller_worker = optional(map(any))
-      worker            = optional(map(any))
+
+      controller = optional(object({
+        ami_id        = optional(string)
+        instance_type = optional(string)
+        os_type       = optional(string)
+
+        volume = optional(object({
+          size = number
+        }))
+
+        user_data    = optional(string)
+        ready_script = optional(string)
+
+        connection = optional(object({
+          type     = string
+          username = string
+        }))
+      }))
+
+      controller_worker = optional(object({
+        ami_id        = optional(string)
+        instance_type = optional(string)
+        os_type       = optional(string)
+
+        volume = optional(object({
+          size = number
+        }))
+
+        user_data    = optional(string)
+        ready_script = optional(string)
+
+        connection = optional(object({
+          type     = string
+          username = string
+        }))
+      }))
+
+      worker = optional(object({
+        ami_id        = optional(string)
+        instance_type = optional(string)
+        os_type       = optional(string)
+
+        volume = optional(object({
+          size = number
+        }))
+
+        user_data    = optional(string)
+        ready_script = optional(string)
+
+        connection = optional(object({
+          type     = string
+          username = string
+        }))
+      }))
     })
   })
 

hack/ostests/modules/k0sctl/config.tf

@@ -39,7 +39,7 @@ locals {
         }),
 
         var.k0s_executable_path == null ? {} : {
-          k0sBinaryPath = var.k0s_executable_path
+          k0sBinaryPath = format("%s%s", var.k0s_executable_path, host.os_type == "windows" ? ".exe" : "")
         },
       )]
     }

hack/ostests/modules/k0sctl/variables.tf

@@ -2,10 +2,11 @@ variable "hosts" {
   type = list(
     object({
       name          = string,
+      os_type       = string,
       role          = string,
       is_controller = bool,
       is_worker     = bool,
-      ipv4          = optional(string),
+      ipv4          = string,
       connection = object({
         type     = string
         username = string

hack/ostests/modules/os/main.tf

@@ -2,28 +2,29 @@ locals {
   # Boilerplate to make OpenTofu a little bit dynamic
 
   os = {
-    al2023      = local.os_al2023
-    alpine_3_17 = local.os_alpine_3_17
-    alpine_3_20 = local.os_alpine_3_20
-    centos_7    = local.os_centos_7
-    centos_8    = local.os_centos_8
-    centos_9    = local.os_centos_9
-    debian_10   = local.os_debian_10
-    debian_11   = local.os_debian_11
-    debian_12   = local.os_debian_12
-    fcos_38     = local.os_fcos_38
-    fedora_38   = local.os_fedora_38
-    flatcar     = local.os_flatcar
-    oracle_7_9  = local.os_oracle_7_9
-    oracle_8_7  = local.os_oracle_8_7
-    oracle_9_1  = local.os_oracle_9_1
-    rhel_7      = local.os_rhel_7
-    rhel_8      = local.os_rhel_8
-    rhel_9      = local.os_rhel_9
-    rocky_8     = local.os_rocky_8
-    rocky_9     = local.os_rocky_9
-    ubuntu_2004 = local.os_ubuntu_2004
-    ubuntu_2204 = local.os_ubuntu_2204
-    ubuntu_2304 = local.os_ubuntu_2304
+    al2023              = local.os_al2023
+    alpine_3_17         = local.os_alpine_3_17
+    alpine_3_20         = local.os_alpine_3_20
+    centos_7            = local.os_centos_7
+    centos_8            = local.os_centos_8
+    centos_9            = local.os_centos_9
+    debian_10           = local.os_debian_10
+    debian_11           = local.os_debian_11
+    debian_12           = local.os_debian_12
+    fcos_38             = local.os_fcos_38
+    fedora_38           = local.os_fedora_38
+    flatcar             = local.os_flatcar
+    oracle_7_9          = local.os_oracle_7_9
+    oracle_8_7          = local.os_oracle_8_7
+    oracle_9_1          = local.os_oracle_9_1
+    rhel_7              = local.os_rhel_7
+    rhel_8              = local.os_rhel_8
+    rhel_9              = local.os_rhel_9
+    rocky_8             = local.os_rocky_8
+    rocky_9             = local.os_rocky_9
+    ubuntu_2004         = local.os_ubuntu_2004
+    ubuntu_2204         = local.os_ubuntu_2204
+    ubuntu_2304         = local.os_ubuntu_2304
+    windows_server_2022 = local.os_windows_server_2022
   }
 }

hack/ostests/modules/os/os_alpine_3_20.tf

@@ -1,7 +1,7 @@
 # https://www.alpinelinux.org/cloud/
 
 data "aws_ami" "alpine_3_20" {
-  count = var.os == "alpine_3_20" ? 1 : 0
+  count = (var.os == "alpine_3_20" || startswith(var.os, "windows_")) ? 1 : 0
 
   owners      = ["538276064493"]
   name_regex  = "^alpine-3\\.20\\.\\d+-(aarch64|x86_64)-uefi-tiny($|-.*)"
@@ -29,7 +29,7 @@ data "aws_ami" "alpine_3_20" {
 }
 
 locals {
-  os_alpine_3_20 = var.os != "alpine_3_20" ? {} : {
+  os_alpine_3_20 = (var.os != "alpine_3_20" && !startswith(var.os, "windows_")) ? {} : {
     node_configs = {
       default = {
         ami_id = one(data.aws_ami.alpine_3_20.*.id)

hack/ostests/modules/os/os_windows_server_2022.tf

@@ -0,0 +1,64 @@
+data "aws_ami" "windows_server_2022" {
+  count = var.os == "windows_server_2022" ? 1 : 0
+
+  owners      = ["801119661308"] # amazon
+  name_regex  = "Windows_Server-2022-English-Full-Base-\\d+\\.\\d+\\.\\d+"
+  most_recent = true
+
+  filter {
+    name   = "name"
+    values = ["Windows_Server-2022-English-Full-Base-*"]
+  }
+
+  filter {
+    name   = "architecture"
+    values = ["x86_64"]
+  }
+
+  filter {
+    name   = "root-device-type"
+    values = ["ebs"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  lifecycle {
+    precondition {
+      condition     = var.arch == "x86_64"
+      error_message = "Unsupported architecture for Windows Server 2022."
+    }
+  }
+}
+
+locals {
+  os_windows_server_2022 = var.os != "windows_server_2022" ? {} : {
+    node_configs = {
+      default    = local.os_alpine_3_20.node_configs.default
+      controller = local.os_alpine_3_20.node_configs.controller
+
+      worker = {
+        ami_id        = one(data.aws_ami.windows_server_2022.*.id)
+        instance_type = "t3a.medium"
+        os_type       = "windows"
+        volume        = { size = 50 }
+
+        # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/user-data.html#user-data-yaml-scripts
+        # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2launch-v2-task-definitions.html#ec2launch-v2-enableopenssh
+        # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2launch-v2.html#ec2launch-v2-directory
+        # C:\ProgramData\Amazon\EC2Launch\log\agent.log
+        user_data = jsonencode({ version = 1.1, tasks = [{ task = "enableOpenSsh" }]})
+
+        # Override the default Alpine ready script. Also checks SSH connectivity.
+        ready_script = "whoami"
+
+        connection = {
+          type     = "ssh"
+          username = "Administrator"
+        }
+      }
+    }
+  }
+}