From 098a29740f98e164b4527fdbde8db6cc9af0ab92 Mon Sep 17 00:00:00 2001 From: "Thomas S. Pangborn" <32514746+thpang@users.noreply.github.com> Date: Thu, 7 Jul 2022 08:03:54 -0400 Subject: [PATCH 1/7] (IAC-592) kube-vip updates (#7) --- docs/CONFIG-VARS.md | 3 +-- docs/REQUIREMENTS.md | 1 - examples/vsphere/sample-terraform-minimal.tfvars | 1 - examples/vsphere/sample-terraform-vi.tfvars | 1 - examples/vsphere/sample-terraform.tfvars | 1 - main.tf | 1 - .../kubernetes/loadbalancer/kube_vip/primary/tasks/main.yaml | 4 ++-- templates/ansible/ansible-vars.yaml.tmpl | 1 - variables.tf | 5 ----- 9 files changed, 3 insertions(+), 15 deletions(-) diff --git a/docs/CONFIG-VARS.md b/docs/CONFIG-VARS.md index 78c2198..e3e4774 100644 --- a/docs/CONFIG-VARS.md +++ b/docs/CONFIG-VARS.md @@ -81,8 +81,7 @@ cluster_domain | Cluster domain suffix for DNS | string | | | | Name | Description | Type | Default | Notes | | :--- | :--- | :--- | :--- | :--- | -kube_vip_version | kube-vip version | string | "0.4.4" | | -kube_vip_interface | kube-vip interface | string | | | +kube_vip_version | kube-vip version | string | "0.4.4" | The minimal supported version is 0.4.4 | kube_vip_ip | kube-vip IP address | string | | | kube_vip_dns | kube-vip DNS | string | | | kube_vip_range | kube-vip IP address range | string | | | diff --git a/docs/REQUIREMENTS.md b/docs/REQUIREMENTS.md index 747344d..4090778 100644 --- a/docs/REQUIREMENTS.md +++ b/docs/REQUIREMENTS.md @@ -148,7 +148,6 @@ cluster_domain = "sample.domain.foo.com" # Cluster domain suffix # Kubernetes - Cluster Virtual IP Address and Cloud Provider kube_vip_version = "0.4.4" -kube_vip_interface = "ens160" kube_vip_ip = "10.18.0.175" kube_vip_dns = "vm-dev-oss-vip.sample.domain.foo.com" kube_vip_range = "10.18.0.100-10.18.0.125" diff --git a/examples/vsphere/sample-terraform-minimal.tfvars b/examples/vsphere/sample-terraform-minimal.tfvars index 0f13df9..6886163 100644 --- a/examples/vsphere/sample-terraform-minimal.tfvars +++ b/examples/vsphere/sample-terraform-minimal.tfvars @@ -28,7 +28,6 @@ cluster_domain = "" # Cluster domain suffix for DNS # Kubernetes - Cluster VIP and Cloud Provider kube_vip_version = "0.4.4" -kube_vip_interface = "" kube_vip_ip = "" kube_vip_dns = "" kube_vip_range = "" diff --git a/examples/vsphere/sample-terraform-vi.tfvars b/examples/vsphere/sample-terraform-vi.tfvars index 12e0d8e..c033047 100644 --- a/examples/vsphere/sample-terraform-vi.tfvars +++ b/examples/vsphere/sample-terraform-vi.tfvars @@ -28,7 +28,6 @@ cluster_domain = "" # Cluster domain suffix for DNS # Kubernetes - Cluster VIP and Cloud Provider kube_vip_version = "0.4.4" -kube_vip_interface = "" kube_vip_ip = "" kube_vip_dns = "" kube_vip_range = "" diff --git a/examples/vsphere/sample-terraform.tfvars b/examples/vsphere/sample-terraform.tfvars index 3a87857..c437c38 100644 --- a/examples/vsphere/sample-terraform.tfvars +++ b/examples/vsphere/sample-terraform.tfvars @@ -28,7 +28,6 @@ cluster_domain = "" # Cluster domain suffix for DNS # Kubernetes - Cluster VIP and Cloud Provider kube_vip_version = "0.4.4" -kube_vip_interface = "" kube_vip_ip = "" kube_vip_dns = "" kube_vip_range = "" diff --git a/main.tf b/main.tf index 8cdd864..17a2219 100644 --- a/main.tf +++ b/main.tf @@ -183,7 +183,6 @@ resource "local_file" "ansible_vars" { cluster_pod_subnet = var.cluster_pod_subnet control_plane_ssh_key_name = var.control_plane_ssh_key_name kube_vip_version = var.kube_vip_version - kube_vip_interface = var.kube_vip_interface # NOTE: Cannot be a loopback interface. Must be the same on all machines kube_vip_ip = var.kube_vip_ip kube_vip_dns = var.kube_vip_dns == null ? "${local.cluster_name}-vip.${var.cluster_domain}" : length(var.kube_vip_dns) > 0 ? var.kube_vip_dns : "${local.cluster_name}-vip.${var.cluster_domain}" kube_vip_range = var.kube_vip_range diff --git a/roles/kubernetes/loadbalancer/kube_vip/primary/tasks/main.yaml b/roles/kubernetes/loadbalancer/kube_vip/primary/tasks/main.yaml index 5a5c712..2a2bc72 100644 --- a/roles/kubernetes/loadbalancer/kube_vip/primary/tasks/main.yaml +++ b/roles/kubernetes/loadbalancer/kube_vip/primary/tasks/main.yaml @@ -17,7 +17,7 @@ - name: Generate kube-vip manifest using ARP (Layer 2) protocol on kubernetes - containerd ansible.builtin.shell: | ctr run --rm --net-host ghcr.io/kube-vip/kube-vip:v{{ kubernetes_vip_version }} vip /kube-vip manifest pod \ - --interface {{ kubernetes_vip_interface }} \ + --enableLoadBalancer \ --address {{ kubernetes_vip_ip }} \ --controlplane \ --services \ @@ -30,7 +30,7 @@ - name: Generate kube-vip manifest using ARP (Layer 2) protocol on kubernetes - docker ansible.builtin.shell: | docker run --network host --rm ghcr.io/kube-vip/kube-vip:v{{ kubernetes_vip_version }} manifest pod \ - --interface {{ kubernetes_vip_interface }} \ + --enableLoadBalancer \ --address {{ kubernetes_vip_ip }} \ --controlplane \ --services \ diff --git a/templates/ansible/ansible-vars.yaml.tmpl b/templates/ansible/ansible-vars.yaml.tmpl index 3c46f21..b3afaea 100644 --- a/templates/ansible/ansible-vars.yaml.tmpl +++ b/templates/ansible/ansible-vars.yaml.tmpl @@ -36,7 +36,6 @@ kubernetes_pod_subnet : "${ cluster_pod_subnet }" # VIP Cloud Provider IP Range : https://kube-vip.chipzoller.dev/docs/usage/cloud-provider/#the-kube-vip-cloud-provider-configmap # kubernetes_vip_version : "${ kube_vip_version }" -kubernetes_vip_interface : "${ kube_vip_interface }" kubernetes_vip_ip : "${ kube_vip_ip }" kubernetes_vip_loadbalanced_dns : "${ kube_vip_dns }" kubernetes_vip_cloud_provider_range : "${ kube_vip_range }" diff --git a/variables.tf b/variables.tf index 9b454d5..5ad9e1f 100644 --- a/variables.tf +++ b/variables.tf @@ -294,11 +294,6 @@ variable "kube_vip_version" { default = "0.4.4" } -variable "kube_vip_interface" { - type = string - default = null -} - variable "kube_vip_ip" { type = string default = null From 0034acd4024007e9d76eac706709869a23e8b642 Mon Sep 17 00:00:00 2001 From: Adam Smith <18425703+AWSmith0216@users.noreply.github.com> Date: Mon, 11 Jul 2022 17:28:45 -0400 Subject: [PATCH 2/7] (IAC-605) Don't replace GRUB_CMDLINE_LINUX if it already exists (#10) --- roles/kubernetes/common/tasks/main.yaml | 26 +++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/roles/kubernetes/common/tasks/main.yaml b/roles/kubernetes/common/tasks/main.yaml index 8aace35..97d3612 100644 --- a/roles/kubernetes/common/tasks/main.yaml +++ b/roles/kubernetes/common/tasks/main.yaml @@ -37,16 +37,38 @@ - name: Enable cgroup v2 # https://rootlesscontaine.rs/getting-started/common/cgroup2/#enabling-cgroup-v2 block: - - name: Adding GRUB_CMDLINE_LINUX to /etc/default/grub file + - name: Retrieve contents of /etc/default/grub + ansible.builtin.shell: cat /etc/default/grub + register: grub_content + changed_when: False + + - name: Retrieve existing GRUB_CMDLINE_LINUX options, if any + set_fact: + grub_cmdline_original: "{{ grub_content.stdout | regex_search('^GRUB_CMDLINE_LINUX=\"([^\"]*)\"$', '\\1', multiline=True) }}" + + - name: Set {{ option }}={{ value }} if option previously disabled + set_fact: + grub_cmdline_updated: "{{ grub_cmdline_original.0 | default('') | replace(option + '=0', option + '=' + value) }}" + + - name: Adding {{ option }}={{ value }} if option not present + set_fact: + grub_cmdline_updated: '{{ (grub_cmdline_updated + " " + option + "=" + value) | trim }}' + when: 'option not in grub_cmdline_updated' + + - name: Adding updated GRUB_CMDLINE_LINUX to /etc/default/grub file ansible.builtin.lineinfile: path: /etc/default/grub regexp: '^GRUB_CMDLINE_LINUX=' - line: GRUB_CMDLINE_LINUX="systemd.unified_cgroup_hierarchy=1" + line: 'GRUB_CMDLINE_LINUX="{{ grub_cmdline_updated }}"' owner: root group: root mode: '0644' + - name: Update GRUB ansible.builtin.command: update-grub + vars: + option: systemd.unified_cgroup_hierarchy + value: "1" tags: - install From 3be0fe17ffc001a13c0f0557cecdfb1cc8a96b3e Mon Sep 17 00:00:00 2001 From: "Thomas S. Pangborn" <32514746+thpang@users.noreply.github.com> Date: Tue, 12 Jul 2022 21:41:50 -0400 Subject: [PATCH 3/7] (IAC-594) node_pools items can now add in n number of misc disks (#9) --- docs/REQUIREMENTS.md | 28 ++- ...rm.tfvars => sample-terraform-dhcp.tfvars} | 46 +++-- .../vsphere/sample-terraform-minimal.tfvars | 33 ++-- .../sample-terraform-static-ips.tfvars | 171 ++++++++++++++++++ examples/vsphere/sample-terraform-vi.tfvars | 10 +- main.tf | 33 +++- modules/vm/main.tf | 15 +- modules/vm/variables.tf | 5 + playbooks/kubernetes-install.yaml | 1 + .../init/primary/templates/kubeadm-config.j2 | 14 -- .../kubernetes/node/baseline/tasks/main.yaml | 1 + .../node/labels_taints/roles/tasks/main.yaml | 2 +- .../node/labels_taints/system/tasks/main.yaml | 4 +- .../node/labels_taints/tasks/labels.yaml | 2 +- .../node/labels_taints/tasks/taints.yaml | 2 +- roles/kubernetes/node/removal/tasks/main.yaml | 4 +- .../defaults/main.yaml | 61 +++++++ .../tasks/main.yaml | 40 ++++ .../vsphere/init/files/create_partitions.sh | 16 ++ roles/systems/vsphere/init/files/link_devs.sh | 23 +++ roles/systems/vsphere/init/tasks/main.yaml | 47 +++++ templates/ansible/ansible-vars.yaml.tmpl | 5 + templates/ansible/inventory.tmpl | 23 ++- variables.tf | 30 ++- 24 files changed, 545 insertions(+), 71 deletions(-) rename examples/vsphere/{sample-terraform.tfvars => sample-terraform-dhcp.tfvars} (88%) create mode 100644 examples/vsphere/sample-terraform-static-ips.tfvars create mode 100644 roles/kubernetes/storage/sig-storage-local-static-provisioner/defaults/main.yaml create mode 100644 roles/kubernetes/storage/sig-storage-local-static-provisioner/tasks/main.yaml create mode 100644 roles/systems/vsphere/init/files/create_partitions.sh create mode 100644 roles/systems/vsphere/init/files/link_devs.sh diff --git a/docs/REQUIREMENTS.md b/docs/REQUIREMENTS.md index 4090778..cb0ece1 100644 --- a/docs/REQUIREMENTS.md +++ b/docs/REQUIREMENTS.md @@ -180,7 +180,7 @@ node_pools = { count = 3 cpus = 2 memory = 4096 - disk = 100 + os_disk = 100 node_taints = [] node_labels = {} }, @@ -191,7 +191,7 @@ node_pools = { count = 1 cpus = 8 memory = 16384 - disk = 100 + os_disk = 100 node_taints = [] node_labels = { "kubernetes.azure.com/mode" = "system" # REQUIRED LABEL - DO NOT REMOVE @@ -201,7 +201,11 @@ node_pools = { count = 3 cpus = 16 memory = 131072 - disk = 350 + os_disk = 350 + misc_disks = [ + 150, + 150, + ] node_taints = ["workload.sas.com/class=cas:NoSchedule"] node_labels = { "workload.sas.com/class" = "cas" @@ -211,7 +215,7 @@ node_pools = { count = 1 cpus = 16 memory = 131072 - disk = 100 + os_disk = 100 node_taints = ["workload.sas.com/class=compute:NoSchedule"] node_labels = { "workload.sas.com/class" = "compute" @@ -222,7 +226,10 @@ node_pools = { count = 1 cpus = 8 memory = 32768 - disk = 100 + os_disk = 100 + misc_disks = [ + 150, + ] node_taints = ["workload.sas.com/class=stateful:NoSchedule"] node_labels = { "workload.sas.com/class" = "stateful" @@ -232,7 +239,10 @@ node_pools = { count = 2 cpus = 8 memory = 32768 - disk = 100 + os_disk = 100 + misc_disks = [ + 150, + ] node_taints = ["workload.sas.com/class=stateless:NoSchedule"] node_labels = { "workload.sas.com/class" = "stateless" @@ -490,6 +500,12 @@ The following items **MUST** be added to your `ansible-vars.yaml` file if you ar ### Ingress Controller INGRESS_NGINX_CONFIG: + controller: + service: + externalTrafficPolicy: Cluster + # loadBalancerIP: # Assigns a specific IP for your loadBalancer + loadBalancerSourceRanges: [] # Not supported on open source kubernetes + annotations: ### Metrics Server METRICS_SERVER_CHART_VERSION: 5.10.14 diff --git a/examples/vsphere/sample-terraform.tfvars b/examples/vsphere/sample-terraform-dhcp.tfvars similarity index 88% rename from examples/vsphere/sample-terraform.tfvars rename to examples/vsphere/sample-terraform-dhcp.tfvars index c437c38..813a711 100644 --- a/examples/vsphere/sample-terraform.tfvars +++ b/examples/vsphere/sample-terraform-dhcp.tfvars @@ -1,9 +1,9 @@ # General items ansible_user = "" ansible_password = "" -prefix = "viya4-k8s" # Infra prefix -gateway = "" # Gateway for servers -netmask = "" # Needed for any network outside the 10.12.0 location +prefix = "v4-k8s-dhcp" # Infra prefix +gateway = "" # Gateway for servers +netmask = "" # Needed for any network outside the 10.12.0 location # vSphere vsphere_server = "" # Name of the vSphere server @@ -55,7 +55,7 @@ node_pools = { count = 3 cpus = 2 memory = 4096 - disk = 100 + os_disk = 100 node_taints = [] node_labels = {} }, @@ -65,17 +65,21 @@ node_pools = { count = 1 cpus = 8 memory = 16384 - disk = 100 + os_disk = 100 node_taints = [] node_labels = { "kubernetes.azure.com/mode" = "system" # REQUIRED LABEL - DO NOT REMOVE } }, cas = { - count = 3 - cpus = 16 - memory = 131072 - disk = 350 + count = 3 + cpus = 16 + memory = 131072 + os_disk = 350 + misc_disks = [ + 150, + 150, + ] node_taints = ["workload.sas.com/class=cas:NoSchedule"] node_labels = { "workload.sas.com/class" = "cas" @@ -85,7 +89,7 @@ node_pools = { count = 1 cpus = 16 memory = 131072 - disk = 100 + os_disk = 100 node_taints = ["workload.sas.com/class=compute:NoSchedule"] node_labels = { "workload.sas.com/class" = "compute" @@ -93,20 +97,26 @@ node_pools = { } }, stateful = { - count = 1 - cpus = 8 - memory = 32768 - disk = 100 + count = 1 + cpus = 8 + memory = 32768 + os_disk = 100 + misc_disks = [ + 150, + ] node_taints = ["workload.sas.com/class=stateful:NoSchedule"] node_labels = { "workload.sas.com/class" = "stateful" } }, stateless = { - count = 2 - cpus = 8 - memory = 32768 - disk = 100 + count = 2 + cpus = 8 + memory = 32768 + os_disk = 100 + misc_disks = [ + 150, + ] node_taints = ["workload.sas.com/class=stateless:NoSchedule"] node_labels = { "workload.sas.com/class" = "stateless" diff --git a/examples/vsphere/sample-terraform-minimal.tfvars b/examples/vsphere/sample-terraform-minimal.tfvars index 6886163..650eafe 100644 --- a/examples/vsphere/sample-terraform-minimal.tfvars +++ b/examples/vsphere/sample-terraform-minimal.tfvars @@ -1,9 +1,9 @@ # General items ansible_user = "" ansible_password = "" -prefix = "viya4-k8s" # Infra prefix -gateway = "" # Gateway for servers -netmask = "" # Needed for any network outside the 10.12.0 location +prefix = "v4-k8s-min" # Infra prefix +gateway = "" # Gateway for servers +netmask = "" # Needed for any network outside the 10.12.0 location # vSphere vsphere_server = "" # Name of the vSphere server @@ -56,7 +56,7 @@ node_pools = { count = 1 cpus = 2 memory = 4096 - disk = 100 + os_disk = 100 node_taints = [] node_labels = {} }, @@ -66,27 +66,34 @@ node_pools = { count = 1 cpus = 8 memory = 16384 - disk = 100 + os_disk = 100 node_taints = [] node_labels = { "kubernetes.azure.com/mode" = "system" # REQUIRED LABEL - DO NOT REMOVE } }, cas = { - count = 3 - cpus = 8 - memory = 16384 - disk = 100 + count = 3 + cpus = 8 + memory = 16384 + os_disk = 100 + misc_disks = [ + 150, + 150, + ] node_taints = ["workload.sas.com/class=cas:NoSchedule"] node_labels = { "workload.sas.com/class" = "cas" } }, generic = { - count = 5 - cpus = 24 # 16 - memory = 131072 - disk = 350 # 250 + count = 5 + cpus = 24 + memory = 131072 + os_disk = 350 + misc_disks = [ + 150, + ] node_taints = [] node_labels = { "workload.sas.com/class" = "compute" diff --git a/examples/vsphere/sample-terraform-static-ips.tfvars b/examples/vsphere/sample-terraform-static-ips.tfvars new file mode 100644 index 0000000..7db12d8 --- /dev/null +++ b/examples/vsphere/sample-terraform-static-ips.tfvars @@ -0,0 +1,171 @@ +# General items +ansible_user = "" +ansible_password = "" +prefix = "v4-k8s-static" # Infra prefix +gateway = "" # Gateway for servers +netmask = "" # Needed for any network outside the 10.12.0 location + +# vSphere +vsphere_server = "" # Name of the vSphere server +vsphere_cluster = "" # Name of the vSphere cluster +vsphere_datacenter = "" # Name of the vSphere data center +vsphere_datastore = "" # Name of the vSphere data store to use for the VMs +vsphere_resource_pool = "" # Name of the vSphere resource pool to use for the VMs +vsphere_folder = "" # Name of the vSphere folder to store the vms +vsphere_template = "" # Name of the VM template to clone to create VMs for the cluster +vsphere_network = "" # Name of the network to to use for the VMs + +# Systems +system_ssh_keys_dir = "~/.ssh/oss" # Directory holding public keys to be used on each system + +# Kubernetes - Cluster +cluster_version = "1.22.10" # Kubernetes Version +cluster_cni = "calico" # Kuberentes Container Network Interface (CNI) +cluster_cri = "containerd" # Kubernetes Container Runtime Interface (CRI) +cluster_service_subnet = "10.43.0.0/16" # Kubernetes Service Subnet +cluster_pod_subnet = "10.42.0.0/16" # Kubernetes Pod Subnet +cluster_domain = "" # Cluster domain suffix for DNS + +# Kubernetes - Cluster VIP and Cloud Provider +kube_vip_version = "0.4.4" +kube_vip_interface = "" +kube_vip_ip = "" +kube_vip_dns = "" +kube_vip_range = "" + +# Control plane node shared ssh key name +control_plane_ssh_key_name = "cp_ssh" + +# Cluster Node Pools config +# +# Your node pools must contain at least 3 or more nodes. +# The required node types are: +# +# * control_plane - Having an odd number 3/5/7... ensures +# HA while using kube-vip +# * system - System node pool to run misc pods, etc +# * cas - CAS Nodes +# * - Any number of node types with unique names. +# These are typically: compute, stateful, and +# stateless. +# +node_pools = { + # REQUIRED NODE TYPE - DO NOT REMOVE and DO NOT CHANGE THE NAME + # Other varaibles may be altered + control_plane = { + cpus = 2 + memory = 4096 + os_disk = 100 + ip_addresses = [ + "", + "", + "", + ] + node_taints = [] + node_labels = {} + }, + # REQUIRED NODE TYPE - DO NOT REMOVE and DO NOT CHANGE THE NAME + # Other varaibles may be altered + system = { + cpus = 8 + memory = 16384 + os_disk = 100 + ip_addresses = [ + "", + ] + node_taints = [] + node_labels = { + "kubernetes.azure.com/mode" = "system" # REQUIRED LABEL - DO NOT REMOVE + } + }, + cas = { + cpus = 16 + memory = 131072 + os_disk = 350 + misc_disks = [ + 150, + 150, + ] + ip_addresses = [ + "", + "", + "", + ] + node_taints = ["workload.sas.com/class=cas:NoSchedule"] + node_labels = { + "workload.sas.com/class" = "cas" + } + }, + compute = { + cpus = 16 + memory = 131072 + os_disk = 100 + ip_addresses = [ + "", + ] + node_taints = ["workload.sas.com/class=compute:NoSchedule"] + node_labels = { + "workload.sas.com/class" = "compute" + "launcher.sas.com/prepullImage" = "sas-programming-environment" + } + }, + stateful = { + cpus = 8 + memory = 32768 + os_disk = 100 + misc_disks = [ + 150, + ] + ip_addresses = [ + "", + ] + node_taints = ["workload.sas.com/class=stateful:NoSchedule"] + node_labels = { + "workload.sas.com/class" = "stateful" + } + }, + stateless = { + cpus = 8 + memory = 32768 + os_disk = 100 + misc_disks = [ + 150, + ] + ip_addresses = [ + "", + "", + ] + node_taints = ["workload.sas.com/class=stateless:NoSchedule"] + node_labels = { + "workload.sas.com/class" = "stateless" + } + } +} + +# Jump server +create_jump = true # Creation flag +jump_num_cpu = 4 # 4 CPUs +jump_memory = 8092 # 8 GB +jump_disk_size = 100 # 100 GB +jump_ip = "10.12.50.30" # Assigned values for static IPs + +# NFS server +create_nfs = true # Creation flag +nfs_num_cpu = 8 # 8 CPUs +nfs_memory = 16384 # 16 GB +nfs_disk_size = 500 # 500 GB +nfs_ip = "10.12.50.31" # Assigned values for static IPs + +# Postgres Servers +postgres_servers = { + default = { + server_num_cpu = 8 # 8 CPUs + server_memory = 16384 # 16 GB + server_disk_size = 250 # 256 GB + server_ip = "10.12.50.32" # Assigned values for static IPs + server_version = 12 # PostgreSQL version + server_ssl = "off" # SSL flag + administrator_login = "postgres" # PostgreSQL admin user - CANNOT BE CHANGED + administrator_password = "my$up3rS3cretPassw0rd" # PostgreSQL admin user password + } +} diff --git a/examples/vsphere/sample-terraform-vi.tfvars b/examples/vsphere/sample-terraform-vi.tfvars index c033047..b962684 100644 --- a/examples/vsphere/sample-terraform-vi.tfvars +++ b/examples/vsphere/sample-terraform-vi.tfvars @@ -55,7 +55,7 @@ node_pools = { count = 1 cpus = 2 memory = 4096 - disk = 100 + os_disk = 100 node_taints = [] node_labels = {} }, @@ -65,7 +65,7 @@ node_pools = { count = 1 cpus = 8 memory = 16384 - disk = 100 + os_disk = 100 node_taints = [] node_labels = { "kubernetes.azure.com/mode" = "system" # REQUIRED LABEL - DO NOT REMOVE @@ -75,7 +75,7 @@ node_pools = { count = 3 cpus = 8 memory = 16384 - disk = 100 + os_disk = 100 node_taints = ["workload.sas.com/class=cas:NoSchedule"] node_labels = { "workload.sas.com/class" = "cas" @@ -83,9 +83,9 @@ node_pools = { }, generic = { count = 5 - cpus = 24 # 16 + cpus = 24 memory = 131072 - disk = 350 # 250 + os_disk = 350 node_taints = [] node_labels = { "workload.sas.com/class" = "compute" diff --git a/main.tf b/main.tf index 17a2219..e9d98da 100644 --- a/main.tf +++ b/main.tf @@ -30,7 +30,8 @@ module "control_plane" { instance_count = length(each.value.ip_addresses) != 0 ? length(each.value.ip_addresses) : each.value.count num_cpu = each.value.cpus memory = each.value.memory - disk_size = each.value.disk + disk_size = each.value.os_disk + misc_disks = each.value.misc_disks ip_addresses = length(each.value.ip_addresses) != 0 ? each.value.ip_addresses : [] } @@ -56,7 +57,8 @@ module "system" { instance_count = length(each.value.ip_addresses) != 0 ? length(each.value.ip_addresses) : each.value.count num_cpu = each.value.cpus memory = each.value.memory - disk_size = each.value.disk + disk_size = each.value.os_disk + misc_disks = each.value.misc_disks ip_addresses = length(each.value.ip_addresses) != 0 ? each.value.ip_addresses : [] } @@ -82,7 +84,8 @@ module "node" { instance_count = length(each.value.ip_addresses) != 0 ? length(each.value.ip_addresses) : each.value.count num_cpu = each.value.cpus memory = each.value.memory - disk_size = each.value.disk + disk_size = each.value.os_disk + misc_disks = each.value.misc_disks ip_addresses = length(each.value.ip_addresses) != 0 ? each.value.ip_addresses : [] } @@ -131,6 +134,28 @@ module "nfs" { dns_servers = var.dns_servers } +module "cr" { + source = "./modules/vm" + + name = "cr" + instance_count = var.create_cr ? 1 : 0 + resource_pool_id = data.vsphere_resource_pool.pool.id + folder = var.vsphere_folder + datastore = var.vsphere_datastore + network = var.vsphere_network + datacenter_id = data.vsphere_datacenter.dc.id + template = var.vsphere_template + cluster_domain = var.cluster_domain + cluster_name = local.cluster_name + ip_addresses = [var.cr_ip] + memory = var.cr_memory + num_cpu = var.cr_num_cpu + disk_size = var.cr_disk_size + netmask = var.netmask + gateway = var.gateway + dns_servers = var.dns_servers +} + module "postgresql" { source = "./modules/server" @@ -162,6 +187,7 @@ resource "local_file" "inventory" { node_ips = length(local.node_ips) > 0 ? local.node_ips : [] nfs_ip = var.create_nfs ? var.nfs_ip : null jump_ip = var.create_jump ? var.jump_ip : null + cr_ip = var.create_cr ? var.cr_ip : null postgres_servers = local.postgres_servers } ) @@ -188,6 +214,7 @@ resource "local_file" "ansible_vars" { kube_vip_range = var.kube_vip_range nfs_ip = var.create_nfs ? var.nfs_ip : null jump_ip = var.create_jump ? var.jump_ip : null + cr_ip = var.create_cr ? var.cr_ip : null system_ssh_keys_dir = var.system_ssh_keys_dir node_labels = local.node_labels node_taints = local.node_taints diff --git a/modules/vm/main.tf b/modules/vm/main.tf index 91f3d09..4f78039 100644 --- a/modules/vm/main.tf +++ b/modules/vm/main.tf @@ -38,9 +38,22 @@ resource "vsphere_virtual_machine" "static" { } disk { - label = "disk0" + label = "os-disk-01" size = var.disk_size thin_provisioned = data.vsphere_virtual_machine.template.disks.0.thin_provisioned + unit_number = 0 + } + dynamic "disk" { + # for_each = var.misc_disks != null ? length(var.misc_disks) > 0 ? var.misc_disks : [] : [] + for_each = var.misc_disks != null ? length(var.misc_disks) > 0 ? { for k, v in var.misc_disks : k => v } : {} : {} + content { + # label = "misc-disk-${index(var.misc_disks, disk.value) + 1}" + label = format("misc-disk-%02d", disk.key + 1) + size = disk.value + thin_provisioned = true + # unit_number = index(var.misc_disks, disk.value) + 1 + unit_number = disk.key + 1 + } } clone { diff --git a/modules/vm/variables.tf b/modules/vm/variables.tf index 9cbbc7f..fc95361 100644 --- a/modules/vm/variables.tf +++ b/modules/vm/variables.tf @@ -62,6 +62,11 @@ variable "disk_size" { type = string } +variable "misc_disks" { + type = list(any) + default = null +} + variable "dns_servers" { type = list(any) } diff --git a/playbooks/kubernetes-install.yaml b/playbooks/kubernetes-install.yaml index b410751..3a5d789 100644 --- a/playbooks/kubernetes-install.yaml +++ b/playbooks/kubernetes-install.yaml @@ -81,6 +81,7 @@ # Setup default storage class for cluster - hosts: localhost roles: + - { role: kubernetes/storage/sig-storage-local-static-provisioner } - { role: kubernetes/storage/nfs-subdir-external-provisioner } # Misc cluster related items diff --git a/roles/kubernetes/control_plane/init/primary/templates/kubeadm-config.j2 b/roles/kubernetes/control_plane/init/primary/templates/kubeadm-config.j2 index 27616e1..c50da17 100644 --- a/roles/kubernetes/control_plane/init/primary/templates/kubeadm-config.j2 +++ b/roles/kubernetes/control_plane/init/primary/templates/kubeadm-config.j2 @@ -26,20 +26,6 @@ networking: podSubnet: "{{ kubernetes_pod_subnet }}" clusterName: "{{ kubernetes_cluster_name }}" --- -apiVersion: kubeadm.k8s.io/v1beta2 -kind: MasterConfiguration -api: - advertiseAddress: "{{ kubernetes_vip_ip }}" -networking: - podSubnet: "{{ kubernetes_pod_subnet }}" -apiServerExtraArgs: - service-node-port-range: 80-32767 - feature-gates: "PersistentLocalVolumes=true,VolumeScheduling=true,MountPropagation=true" -controllerManagerExtraArgs: - feature-gates: "PersistentLocalVolumes=true,VolumeScheduling=true,MountPropagation=true" -schedulerExtraArgs: - feature-gates: "PersistentLocalVolumes=true,VolumeScheduling=true,MountPropagation=true" ---- apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration authentication: diff --git a/roles/kubernetes/node/baseline/tasks/main.yaml b/roles/kubernetes/node/baseline/tasks/main.yaml index 95de6a3..ee955f3 100644 --- a/roles/kubernetes/node/baseline/tasks/main.yaml +++ b/roles/kubernetes/node/baseline/tasks/main.yaml @@ -36,6 +36,7 @@ owner: "{{ owner }}" group: "{{ group }}" mode: 0777 + ignore_errors: true tags: - install - update diff --git a/roles/kubernetes/node/labels_taints/roles/tasks/main.yaml b/roles/kubernetes/node/labels_taints/roles/tasks/main.yaml index dbc7b39..4086b55 100644 --- a/roles/kubernetes/node/labels_taints/roles/tasks/main.yaml +++ b/roles/kubernetes/node/labels_taints/roles/tasks/main.yaml @@ -1,7 +1,7 @@ --- - name: Label/Taints for cluster nodes ansible.builtin.shell: | - kubectl label nodes {{ ansible_nodename }} node-role.kubernetes.io/node= --overwrite + kubectl label nodes {{ ansible_hostname }} node-role.kubernetes.io/node= --overwrite delegate_to: "{{groups['k8s_control_plane'][0]}}" tags: - install diff --git a/roles/kubernetes/node/labels_taints/system/tasks/main.yaml b/roles/kubernetes/node/labels_taints/system/tasks/main.yaml index e878fa1..16efcf1 100644 --- a/roles/kubernetes/node/labels_taints/system/tasks/main.yaml +++ b/roles/kubernetes/node/labels_taints/system/tasks/main.yaml @@ -1,8 +1,8 @@ --- - name: Label/Taints for System/Default node ansible.builtin.shell: | - kubectl label nodes {{ ansible_nodename }} kubernetes.azure.com/mode=system --overwrite - kubectl label nodes {{ ansible_nodename }} node-role.kubernetes.io/system-node= --overwrite + kubectl label nodes {{ ansible_hostname }} kubernetes.azure.com/mode=system --overwrite + kubectl label nodes {{ ansible_hostname }} node-role.kubernetes.io/system-node= --overwrite delegate_to: "{{groups['k8s_control_plane'][0]}}" tags: - install diff --git a/roles/kubernetes/node/labels_taints/tasks/labels.yaml b/roles/kubernetes/node/labels_taints/tasks/labels.yaml index c4d12b6..18e664f 100644 --- a/roles/kubernetes/node/labels_taints/tasks/labels.yaml +++ b/roles/kubernetes/node/labels_taints/tasks/labels.yaml @@ -1,7 +1,7 @@ --- - name: Applying labels ansible.builtin.shell: | - kubectl label nodes {{ ansible_nodename }} {{ label }} --overwrite + kubectl label nodes {{ ansible_hostname }} {{ label }} --overwrite with_items: "{{ labels }}" loop_control: loop_var: label diff --git a/roles/kubernetes/node/labels_taints/tasks/taints.yaml b/roles/kubernetes/node/labels_taints/tasks/taints.yaml index e5d84bb..2dad9ca 100644 --- a/roles/kubernetes/node/labels_taints/tasks/taints.yaml +++ b/roles/kubernetes/node/labels_taints/tasks/taints.yaml @@ -1,7 +1,7 @@ --- - name: Applying taints ansible.builtin.shell: | - kubectl taint nodes {{ ansible_nodename }} {{ taint }} --overwrite + kubectl taint nodes {{ ansible_hostname }} {{ taint }} --overwrite with_items: "{{ taints }}" loop_control: loop_var: taint diff --git a/roles/kubernetes/node/removal/tasks/main.yaml b/roles/kubernetes/node/removal/tasks/main.yaml index eb8d4ec..2f6f2f0 100644 --- a/roles/kubernetes/node/removal/tasks/main.yaml +++ b/roles/kubernetes/node/removal/tasks/main.yaml @@ -1,7 +1,7 @@ --- - name: Draining the node ansible.builtin.shell: | - kubectl drain {{ ansible_nodename }} --delete-emptydir-data --force --ignore-daemonsets + kubectl drain {{ ansible_hostname }} --delete-emptydir-data --force --ignore-daemonsets delegate_to: "{{groups['k8s_control_plane'][0]}}" ignore_errors: true tags: @@ -22,7 +22,7 @@ - name: Remove the node ansible.builtin.shell: | - kubectl delete nodes {{ ansible_nodename }} + kubectl delete nodes {{ ansible_hostname }} delegate_to: "{{groups['k8s_control_plane'][0]}}" ignore_errors: true tags: diff --git a/roles/kubernetes/storage/sig-storage-local-static-provisioner/defaults/main.yaml b/roles/kubernetes/storage/sig-storage-local-static-provisioner/defaults/main.yaml new file mode 100644 index 0000000..6d2ee10 --- /dev/null +++ b/roles/kubernetes/storage/sig-storage-local-static-provisioner/defaults/main.yaml @@ -0,0 +1,61 @@ +## sig-storage-local-static-provisioner +LOCAL_VOLUME_NAME: sig-storage-local-static-provisioner-sas +LOCAL_VOLUME_NAMESPACE: kube-system +LOCAL_VOLUME_CHART_NAME: Chart.yaml +LOCAL_VOLUME_REPO: https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner.git +LOCAL_VOLUME_CHART_VERSION: 2.4.0 +LOCAL_VOLUME_REPO_VERSION: "v{{ LOCAL_VOLUME_CHART_VERSION }}" +LOCAL_VOLUME_REPO_LOCATION: "/tmp/{{ LOCAL_VOLUME_NAME }}" +LOCAL_VOLUME_CONFIG: + common: + serviceAccount: + create: true + classes: + - name: local-storage + hostDir: /mnt/sas/volumes + volumeMode: Filesystem + fsType: xfs + namePattern: "*" + blockCleanerCommand: + - "/scripts/shred.sh" + - "2" + storageClass: true + daemonset: + tolerations: + - effect: NoSchedule + key: workload.sas.com/class + operator: Equal + value: stateful + - effect: NoSchedule + key: workload.sas.com/class + operator: Equal + value: stateless + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: workload.sas.com/class + operator: In + values: + - stateful + matchFields: [] + weight: 100 + - preference: + matchExpressions: + - key: workload.sas.com/class + operator: NotIn + values: + - compute + - cas + - stateless + matchFields: [] + weight: 50 + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.azure.com/mode + operator: NotIn + values: + - system + matchFields: [] diff --git a/roles/kubernetes/storage/sig-storage-local-static-provisioner/tasks/main.yaml b/roles/kubernetes/storage/sig-storage-local-static-provisioner/tasks/main.yaml new file mode 100644 index 0000000..4137a1d --- /dev/null +++ b/roles/kubernetes/storage/sig-storage-local-static-provisioner/tasks/main.yaml @@ -0,0 +1,40 @@ +--- +# NOTE: The usage and setting of the iac_inventory_dir variable +# is only needed given this task is run on localhost +# an implied localhost and this keeps the magic inventory_file +# and inventory_dir from having values. +# +# Reference URL : https://docs.ansible.com/ansible/latest/inventory/implicit_localhost.html +# +- name: Cloning sig-storage-local-static-provisioner + ansible.builtin.git: + repo: "{{ LOCAL_VOLUME_REPO }}" + dest: "/tmp/{{ LOCAL_VOLUME_NAME }}" + version: "{{ LOCAL_VOLUME_REPO_VERSION }}" + tags: + - install + - update + +- name: Setting up default storage for cluster using sig-storage-local-static-provisioner + kubernetes.core.helm: + name: "{{ LOCAL_VOLUME_NAME }}" + namespace: "{{ LOCAL_VOLUME_NAMESPACE }}" + chart_ref: "{{ LOCAL_VOLUME_REPO_LOCATION }}/helm/provisioner" + chart_version: "{{ LOCAL_VOLUME_CHART_VERSION }}" + values: "{{ LOCAL_VOLUME_CONFIG }}" + kubeconfig: "{{ iac_inventory_dir }}/{{ kubernetes_cluster_name }}-kubeconfig.conf" + create_namespace: true + wait: true + tags: + - install + - update + +- name: Remove sig-storage-local-static-provisioner + kubernetes.core.helm: + name: "{{ LOCAL_VOLUME_NAME }}" + namespace: "{{ LOCAL_VOLUME_NAMESPACE }}" + kubeconfig: "{{ iac_inventory_dir }}/{{ kubernetes_cluster_name }}-kubeconfig.conf" + wait: true + state: absent + tags: + - uninstall diff --git a/roles/systems/vsphere/init/files/create_partitions.sh b/roles/systems/vsphere/init/files/create_partitions.sh new file mode 100644 index 0000000..f4f650f --- /dev/null +++ b/roles/systems/vsphere/init/files/create_partitions.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +all_disks=($(/usr/bin/lsblk --nodeps --noheadings --output NAME --paths | grep sd)) +for disk in "${all_disks[@]}"; do + partitions="$(/usr/bin/lsblk --noheadings --output PTTYPE "${disk}" | grep -vE "^$")" + if [ "${partitions}" == "" ]; then + fdisk ${disk} << EOF +n +p +1 + + +w +EOF + fi +done diff --git a/roles/systems/vsphere/init/files/link_devs.sh b/roles/systems/vsphere/init/files/link_devs.sh new file mode 100644 index 0000000..4b8faba --- /dev/null +++ b/roles/systems/vsphere/init/files/link_devs.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# Find all disks not currently partitioned and add a partition for +all_disks=($(/usr/bin/lsblk --nodeps --noheadings --output NAME --paths | grep sd)) +for disk in "${all_disks[@]}"; do + partitions="$(/usr/bin/lsblk --noheadings --output PTTYPE "${disk}" | grep -vE "^$")" + if [ "${partitions}" == "" ]; then + # Capture the SERIAL number of the disk being used + SERIAL_NUMBER=$(lsblk --noheadings --output SERIAL "${disk}" | grep -vE "^$" ) + if [[ "${disk}" =~ .*\/(.*)$ ]]; then + RDISK="${BASH_REMATCH[1]}" + if [ "${SERIAL_NUMBER}" != "" ]; then + # Verifying the /mnt/sas/volumes directory is present + if [[ ! -d /mnt/sas/volumes/ ]]; then + mkdir -p /mnt/sas/volumes + fi + if [[ ! -L "/mnt/sas/volumes/sas-v4-${RDISK}-${SERIAL_NUMBER}" ]]; then + ln -s "${disk}" "/mnt/sas/volumes/sas-v4-${RDISK}-${SERIAL_NUMBER}" + fi + fi + fi + fi +done diff --git a/roles/systems/vsphere/init/tasks/main.yaml b/roles/systems/vsphere/init/tasks/main.yaml index f0187c2..e06bf84 100644 --- a/roles/systems/vsphere/init/tasks/main.yaml +++ b/roles/systems/vsphere/init/tasks/main.yaml @@ -28,3 +28,50 @@ ansible.builtin.shell: "xfs_growfs -d /" tags: - install + +- name: Set owner and group for Ubuntu OS + ansible.builtin.set_fact: + owner: nobody + group: nogroup + when: ansible_distribution == "Ubuntu" and ansible_distribution_version == "20.04" + tags: + - install + - update + +- name: Create /mnt/sas/volumes directory + ansible.builtin.file: + path: /mnt/sas/volumes + state: directory + owner: "{{ owner }}" + group: "{{ group }}" + mode: 0777 + ignore_errors: true + tags: + - install + - update + +# +# Create local-static-provisioner items for later use +# + +# Copy script +- name: Copy link creation script + ansible.builtin.copy: + src: ./files/link_devs.sh + dest: /tmp/link_devs.sh + owner: root + group: root + mode: '0700' + tags: + - install + +# TODO: Add code here to add partitions for disks +# that are not going to be used for local-static +# storage class storage + +# Run script +- name: Run link creation script + ansible.builtin.shell: | + /tmp/link_devs.sh + tags: + - install diff --git a/templates/ansible/ansible-vars.yaml.tmpl b/templates/ansible/ansible-vars.yaml.tmpl index b3afaea..24aa172 100644 --- a/templates/ansible/ansible-vars.yaml.tmpl +++ b/templates/ansible/ansible-vars.yaml.tmpl @@ -103,3 +103,8 @@ jump_ip : "${jump_ip}" # NFS Server nfs_ip : "${nfs_ip}" %{ endif ~} + +%{ if length(cr_ip) != 0 ~} +# Container Registry +cr_ip : "${cr_ip}" +%{ endif ~} diff --git a/templates/ansible/inventory.tmpl b/templates/ansible/inventory.tmpl index 582e4ce..48231d4 100644 --- a/templates/ansible/inventory.tmpl +++ b/templates/ansible/inventory.tmpl @@ -65,6 +65,20 @@ ${nfs_ip} nfs_server %{ endif ~} +%{ if length(cr_ip) != 0 ~} +# +# Container Registry +# +[cr_server] +${cr_ip} + +# +# Container Registry - alias - DO NOT MODIFY +# +[cr:children] +cr_server +%{ endif ~} + %{ if length(postgres_servers) != 0 ~} # # Postgres Servers @@ -75,8 +89,8 @@ ${server_data.server_ip} [${prefix}_${server_name}_pgsql:vars] postgres_server_name=${server_name} postgres_server_version=${server_data.server_version} -postgres_server_ssl=${server_data.server_ssl} # NOTE: Values - [on,off] -postgres_administrator_login=${server_data.administrator_login} # NOTE: Do not change this value at this time +postgres_server_ssl=${server_data.server_ssl} +postgres_administrator_login=${server_data.administrator_login} postgres_administrator_password=${server_data.administrator_password} %{ endfor ~} @@ -94,7 +108,12 @@ k8s %{ if length(jump_ip) != 0 ~} jump %{ endif ~} +%{ if length(nfs_ip) != 0 ~} nfs +%{ endif ~} +%{ if length(cr_ip) != 0 ~} +cr +%{ endif ~} %{ if length(postgres_servers) != 0 ~} postgres %{ endif ~} diff --git a/variables.tf b/variables.tf index 5ad9e1f..194ce60 100644 --- a/variables.tf +++ b/variables.tf @@ -122,7 +122,8 @@ variable "node_pool_defaults" { default = { cpus = 2 memory = 4096 - disk = 25 + os_disk = 25 + misc_disks = [] count = 0 ip_addresses = [] node_taints = [] @@ -213,6 +214,31 @@ variable "nfs_disk_size" { default = 250 } +# container registry +variable "create_cr" { + type = bool + default = false +} +variable "cr_ip" { + type = string + default = null +} + +variable "cr_memory" { + type = number + default = 8092 +} + +variable "cr_num_cpu" { + type = number + default = 4 +} + +variable "cr_disk_size" { + type = number + default = 160 +} + # postgres variable "postgres_server_defaults" { description = "" @@ -260,7 +286,7 @@ variable "system_ssh_keys_dir" { } variable "cluster_domain" { - type = string + type = string default = null } From 34ef3e3156c9c5972184387937c7d1a32cbb60bf Mon Sep 17 00:00:00 2001 From: Jay Patel <78554593+jarpat@users.noreply.github.com> Date: Thu, 14 Jul 2022 12:37:28 -0400 Subject: [PATCH 4/7] fix: (IAC-621) Update Ansible template files (#22) Co-authored-by: Thomas S. Pangborn --- templates/ansible/ansible-vars.yaml.tmpl | 6 ++++++ templates/ansible/inventory.tmpl | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/templates/ansible/ansible-vars.yaml.tmpl b/templates/ansible/ansible-vars.yaml.tmpl index 24aa172..de4735c 100644 --- a/templates/ansible/ansible-vars.yaml.tmpl +++ b/templates/ansible/ansible-vars.yaml.tmpl @@ -94,17 +94,23 @@ node_taints: %{ endfor ~} %{ endif ~} +%{ if jump_ip != null ~} %{ if length(jump_ip) != 0 ~} # Jump Server jump_ip : "${jump_ip}" %{ endif ~} +%{ endif ~} +%{ if nfs_ip != null ~} %{ if length(nfs_ip) != 0 ~} # NFS Server nfs_ip : "${nfs_ip}" %{ endif ~} +%{ endif ~} +%{ if cr_ip != null ~} %{ if length(cr_ip) != 0 ~} # Container Registry cr_ip : "${cr_ip}" %{ endif ~} +%{ endif ~} diff --git a/templates/ansible/inventory.tmpl b/templates/ansible/inventory.tmpl index 48231d4..5b44ec7 100644 --- a/templates/ansible/inventory.tmpl +++ b/templates/ansible/inventory.tmpl @@ -37,6 +37,7 @@ FIXME - ENTER YOUR KUBERNETES COMPUTE NODE IPs/FQDNs HERE! k8s_control_plane k8s_node +%{ if jump_ip != null ~} %{ if length(jump_ip) != 0 ~} # # Jump Server @@ -50,7 +51,9 @@ ${jump_ip} [jump:children] jump_server %{ endif ~} +%{ endif ~} +%{ if nfs_ip != null ~} %{ if length(nfs_ip) != 0 ~} # # NFS Server @@ -64,7 +67,9 @@ ${nfs_ip} [nfs:children] nfs_server %{ endif ~} +%{ endif ~} +%{ if cr_ip != null ~} %{ if length(cr_ip) != 0 ~} # # Container Registry @@ -78,6 +83,7 @@ ${cr_ip} [cr:children] cr_server %{ endif ~} +%{ endif ~} %{ if length(postgres_servers) != 0 ~} # @@ -105,15 +111,21 @@ ${prefix}_${server_name}_pgsql # [all:children] k8s +%{ if jump_ip != null ~} %{ if length(jump_ip) != 0 ~} jump %{ endif ~} +%{ endif ~} +%{ if nfs_ip != null ~} %{ if length(nfs_ip) != 0 ~} nfs %{ endif ~} +%{ endif ~} +%{ if cr_ip != null ~} %{ if length(cr_ip) != 0 ~} cr %{ endif ~} +%{ endif ~} %{ if length(postgres_servers) != 0 ~} postgres %{ endif ~} From 2cbf06305adfb0b52dccd93d6240dea1877ad77a Mon Sep 17 00:00:00 2001 From: Jay Patel <78554593+jarpat@users.noreply.github.com> Date: Fri, 15 Jul 2022 11:12:12 -0400 Subject: [PATCH 5/7] feat: (IAC-578) SSL Configuration Support for Postgres (#14) --- docs/CONFIG-VARS.md | 8 ++ roles/systems/postgres/defaults/main.yaml | 2 + roles/systems/postgres/tasks/main.yaml | 113 +++++++++++++++++++++- templates/ansible/inventory.tmpl | 12 ++- variables.tf | 2 + 5 files changed, 130 insertions(+), 7 deletions(-) create mode 100644 roles/systems/postgres/defaults/main.yaml diff --git a/docs/CONFIG-VARS.md b/docs/CONFIG-VARS.md index e3e4774..4437e1f 100644 --- a/docs/CONFIG-VARS.md +++ b/docs/CONFIG-VARS.md @@ -254,9 +254,17 @@ nfs_ip = "" # Assigned values for static IP addresses | server_ip | Static IP address for PostgreSQL server | string | | This is a required field | | server_version | PostgreSQL version | number | 12 | | | server_ssl | Enable/disable SSL | string | "off" | | +| server_ssl_cert_file | Path to the PostgreSQL SSL certificate file | string | "" | If `server_ssl` is on and this variable is not defined, the System default SSL cert will be used | +| server_ssl_key_file | Path to the PostgreSQL SSL key file | string | "" | If `server_ssl` is on and this variable is not defined, the System default SSL key will be used | | administrator_login | Admin user | string | "postgres" | | | administrator_password | Admin password | string | "my$up3rS3cretPassw0rd" | | +**NOTES**: + +1. If you set `server_ssl` to on, and you do not define either `server_ssl_cert_file` or `server_ssl_cert_file` the system's default SSL certificate and key will be used instead. By default, on Ubuntu systems we create a copy of those files and name them `ssl-cert-sas.pem` and `ssl-cert-sas.key` + * The Ansible tasks will take care of copying the certificate and key from the Postgres VM into your local workspace directory +2. If you are planning on using the [viya4-deployment repository](https://github.com/sassoftware/viya4-deployment) to perform a Viya deployment where you have [full-stack TLS](https://github.com/sassoftware/viya4-deployment/blob/main/docs/CONFIG-VARS.md#tls), ensure in the viya4-deployment ansible-vars.yaml the `V4_CFG_TLS_TRUSTED_CA_CERTS` variable points to a directory that contains the `server_ssl_cert_file`. + Sample: ```bash diff --git a/roles/systems/postgres/defaults/main.yaml b/roles/systems/postgres/defaults/main.yaml new file mode 100644 index 0000000..68af474 --- /dev/null +++ b/roles/systems/postgres/defaults/main.yaml @@ -0,0 +1,2 @@ +postgres_server_ssl_cert_file: "" +postgres_server_ssl_key_file: "" diff --git a/roles/systems/postgres/tasks/main.yaml b/roles/systems/postgres/tasks/main.yaml index 12fc650..c3587ff 100644 --- a/roles/systems/postgres/tasks/main.yaml +++ b/roles/systems/postgres/tasks/main.yaml @@ -17,7 +17,7 @@ tags: - install - update - + - name: Install postgres ansible.builtin.apt: pkg: @@ -53,15 +53,124 @@ - install - update +- name: Postgres SSL with user supplied cert and key - verify both defined + ansible.builtin.fail: + msg: "User has set postgres_server_ssl to on, but has only defined one of 'postgres_server_ssl_cert_file, + postgres_server_ssl_key_file'. Ensure that both these variables are set, or remove both of them from your + inventory file to use the Ubuntu system default cert and key files." + when: + - postgres_server_ssl|lower == "on" + - (postgres_server_ssl_cert_file|length == 0) and (postgres_server_ssl_key_file|length > 0) + - (postgres_server_ssl_cert_file|length > 0) and (postgres_server_ssl_key_file|length == 0) + tags: + - install + - update + +- name: Copy user supplied cert and key to Postgres host + block: + - name: Add user supplied SSL Cert + ansible.builtin.copy: + src: "{{ postgres_server_ssl_cert_file }}" + dest: /etc/ssl/certs/ + owner: root + group: root + mode: "0644" + register: cert_result + - name: Add user supplied SSL Key + ansible.builtin.copy: + src: "{{ postgres_server_ssl_key_file }}" + dest: /etc/ssl/private/ + owner: root + group: ssl-cert + mode: "0640" + register: key_result + - name: Set postgres_server_ssl_cert_file_dest and postgres_server_ssl_key_file_dest + ansible.builtin.set_fact: + postgres_server_ssl_cert_file_dest: "{{ cert_result.dest }}" + postgres_server_ssl_key_file_dest: "{{ key_result.dest }}" + when: + - ansible_distribution == "Ubuntu" and ansible_distribution_version == "20.04" + - postgres_server_ssl|lower == "on" + - (postgres_server_ssl_cert_file|length > 0) and (postgres_server_ssl_key_file|length > 0) + tags: + - install + - update + +- name: Create copy of Ubuntu default cert and key + block: + - name: Copy Ubuntu default SSL Cert + ansible.builtin.copy: + src: /etc/ssl/certs/ssl-cert-snakeoil.pem + dest: /etc/ssl/certs/ssl-cert-sas.pem + owner: root + group: root + mode: "0644" + remote_src: yes + register: cert_result + - name: Copy Ubuntu default SSL Key + ansible.builtin.copy: + src: /etc/ssl/private/ssl-cert-snakeoil.key + dest: /etc/ssl/private/ssl-cert-sas.key + owner: root + group: ssl-cert + mode: "0640" + remote_src: yes + register: key_result + - name: Fetch SSL Cert + ansible.builtin.fetch: + src: "{{ cert_result.dest }}" + dest: "{{ inventory_dir }}/" + flat: yes + - name: Fetch SSL Key + ansible.builtin.fetch: + src: "{{ key_result.dest }}" + dest: "{{ inventory_dir }}/" + flat: yes + - name: Set postgres_server_ssl_cert_file_dest and postgres_server_ssl_key_file_dest + ansible.builtin.set_fact: + postgres_server_ssl_cert_file_dest: "{{ cert_result.dest }}" + postgres_server_ssl_key_file_dest: "{{ key_result.dest }}" + when: + - ansible_distribution == "Ubuntu" and ansible_distribution_version == "20.04" + - postgres_server_ssl|lower == "on" + - (postgres_server_ssl_cert_file|length == 0) and (postgres_server_ssl_key_file|length == 0) + tags: + - install + - update + - name: Alter postgresql.conf - Update ssl ansible.builtin.lineinfile: path: /etc/postgresql/{{ postgres_server_version }}/main/postgresql.conf - regexp: '^ssl = on' + regexp: '^ssl = (on|off)' line: "ssl = {{ postgres_server_ssl }}" tags: - install - update +- name: Alter postgresql.conf - Update ssl_cert_file + ansible.builtin.lineinfile: + path: /etc/postgresql/{{ postgres_server_version }}/main/postgresql.conf + regexp: '^ssl_cert_file =' + line: "ssl_cert_file = '{{ postgres_server_ssl_cert_file_dest }}'" + when: + - postgres_server_ssl|lower == "on" + - (postgres_server_ssl_cert_file_dest|length > 0) and (postgres_server_ssl_key_file_dest|length > 0) + tags: + - install + - update + +- name: Alter postgresql.conf - Update ssl_key_file + ansible.builtin.lineinfile: + path: /etc/postgresql/{{ postgres_server_version }}/main/postgresql.conf + regexp: '^ssl_key_file =' + line: "ssl_key_file = '{{ postgres_server_ssl_key_file_dest }}'" + when: + - postgres_server_ssl|lower == "on" + - (postgres_server_ssl_cert_file_dest|length > 0) and (postgres_server_ssl_key_file_dest|length > 0) + tags: + - install + - update + # TODO: This should be adjusted to enhance security - name: Alter pg_hba.conf ansible.builtin.lineinfile: diff --git a/templates/ansible/inventory.tmpl b/templates/ansible/inventory.tmpl index 5b44ec7..93c79c6 100644 --- a/templates/ansible/inventory.tmpl +++ b/templates/ansible/inventory.tmpl @@ -93,11 +93,13 @@ cr_server [${prefix}_${server_name}_pgsql] ${server_data.server_ip} [${prefix}_${server_name}_pgsql:vars] -postgres_server_name=${server_name} -postgres_server_version=${server_data.server_version} -postgres_server_ssl=${server_data.server_ssl} -postgres_administrator_login=${server_data.administrator_login} -postgres_administrator_password=${server_data.administrator_password} +postgres_server_name="${server_name}" +postgres_server_version="${server_data.server_version}" +postgres_server_ssl="${server_data.server_ssl}" +postgres_administrator_login="${server_data.administrator_login}" +postgres_administrator_password="${server_data.administrator_password}" +postgres_server_ssl_cert_file="${server_data.server_ssl_cert_file}" +postgres_server_ssl_key_file="${server_data.server_ssl_key_file}" %{ endfor ~} [postgres:children] diff --git a/variables.tf b/variables.tf index 194ce60..ebe38fc 100644 --- a/variables.tf +++ b/variables.tf @@ -250,6 +250,8 @@ variable "postgres_server_defaults" { server_ip = "" # Assigned values for static IPs server_version = 12 # PostgreSQL version server_ssl = "off" # SSL flag + server_ssl_cert_file = "" # PostgreSQL SSL certificate file + server_ssl_key_file = "" # PostgreSQL SSL key file administrator_login = "postgres" # PostgreSQL admin user - CANNOT BE CHANGED administrator_password = "my$up3rS3cretPassw0rd" # PostgreSQL admin user password } From 84263cce234106ad7dd6976b284c530834fa234f Mon Sep 17 00:00:00 2001 From: "Thomas S. Pangborn" <32514746+thpang@users.noreply.github.com> Date: Mon, 18 Jul 2022 13:55:46 -0400 Subject: [PATCH 6/7] (IAC-626) Updating DHCP items for local_storage (#23) --- README.md | 6 +++--- docs/user/DockerUsage.md | 4 ++-- modules/vm/main.tf | 15 ++++++++++++++- .../defaults/main.yaml | 15 ++++++++++++--- 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index ab0d2f0..ae10a06 100644 --- a/README.md +++ b/README.md @@ -89,8 +89,8 @@ A more comprehensive description of these items and their requirements can be fo This project supports the following options for running: -- Running the bash `run.sh` script on your local machine -- Using a Docker container to run the `run.sh` script (Docker is required) +- Running the bash `oss-k8s.sh` script on your local machine +- Using a Docker container to run the `oss-k8s.sh` script (Docker is required) For more information, see [Docker Usage](./docs/user/DockerUsage.md). Using Docker to run the Terraform and Ansible scripts is recommended. @@ -145,7 +145,7 @@ In order to use this repository, modify the [`inventory` file](./examples/bare-m Create and manage the required cluster resources for your SAS Viya 4 deployment. Perform one of the following steps, based on whether you are using Docker: -- Run the [`run.sh`](docs/user/ScriptUsage.md) script directly on your workstation +- Run the [`oss-k8s.sh`](docs/user/ScriptUsage.md) script directly on your workstation - Start the [Docker container](docs/user/DockerUsage.md) (recommended) ## Contributing diff --git a/docs/user/DockerUsage.md b/docs/user/DockerUsage.md index d422883..05fea03 100644 --- a/docs/user/DockerUsage.md +++ b/docs/user/DockerUsage.md @@ -10,7 +10,7 @@ After satisfying all of the prerequisite items that are listed in the [README.md docker build -t viya4-iac-k8s . ``` -The Docker image, `viya4-iac-k8s`, contains Ansible, Terraform, Helm, and kubectl executables. The entrypoint for the Docker image is `run.sh`. The entrypoint is run with subcommands in the subsequent steps. +The Docker image, `viya4-iac-k8s`, contains Ansible, Terraform, Helm, and kubectl executables. The entrypoint for the Docker image is `oss-k8s.sh`. The entrypoint is run with subcommands in the subsequent steps. ### VMware vSphere/vCenter Environment File for Authentication @@ -111,7 +111,7 @@ docker run --rm -it \ ### Display Terraform Outputs - vSphere/vCenter -Once your resources have been created using the `run.sh` command, you can display Terraform output values by running the `viya4-iac-k8s` Docker image with the `output` command: +Once your resources have been created using the `oss-k8s.sh` command, you can display Terraform output values by running the `viya4-iac-k8s` Docker image with the `output` command: ```bash docker run --rm --group-add root \ diff --git a/modules/vm/main.tf b/modules/vm/main.tf index 4f78039..686567c 100644 --- a/modules/vm/main.tf +++ b/modules/vm/main.tf @@ -97,9 +97,22 @@ resource "vsphere_virtual_machine" "dhcp" { } disk { - label = "disk0" + label = "os-disk-01" size = var.disk_size thin_provisioned = data.vsphere_virtual_machine.template.disks.0.thin_provisioned + unit_number = 0 + } + dynamic "disk" { + # for_each = var.misc_disks != null ? length(var.misc_disks) > 0 ? var.misc_disks : [] : [] + for_each = var.misc_disks != null ? length(var.misc_disks) > 0 ? { for k, v in var.misc_disks : k => v } : {} : {} + content { + # label = "misc-disk-${index(var.misc_disks, disk.value) + 1}" + label = format("misc-disk-%02d", disk.key + 1) + size = disk.value + thin_provisioned = true + # unit_number = index(var.misc_disks, disk.value) + 1 + unit_number = disk.key + 1 + } } clone { diff --git a/roles/kubernetes/storage/sig-storage-local-static-provisioner/defaults/main.yaml b/roles/kubernetes/storage/sig-storage-local-static-provisioner/defaults/main.yaml index 6d2ee10..f6759da 100644 --- a/roles/kubernetes/storage/sig-storage-local-static-provisioner/defaults/main.yaml +++ b/roles/kubernetes/storage/sig-storage-local-static-provisioner/defaults/main.yaml @@ -25,11 +25,19 @@ LOCAL_VOLUME_CONFIG: - effect: NoSchedule key: workload.sas.com/class operator: Equal - value: stateful + value: cas - effect: NoSchedule key: workload.sas.com/class operator: Equal value: stateless + - effect: NoSchedule + key: workload.sas.com/class + operator: Equal + value: stateful + - effect: NoSchedule + key: workload.sas.com/class + operator: Equal + value: singlestore affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -39,6 +47,9 @@ LOCAL_VOLUME_CONFIG: operator: In values: - stateful + - stateless + - cas + - singlestore matchFields: [] weight: 100 - preference: @@ -47,8 +58,6 @@ LOCAL_VOLUME_CONFIG: operator: NotIn values: - compute - - cas - - stateless matchFields: [] weight: 50 requiredDuringSchedulingIgnoredDuringExecution: From e4621ebf51879e170aef718def6dfe9ee25f0129 Mon Sep 17 00:00:00 2001 From: Jay Patel <78554593+jarpat@users.noreply.github.com> Date: Thu, 21 Jul 2022 14:45:54 -0400 Subject: [PATCH 7/7] fix: (IAC-578) Update SSL Cert/Key Naming Scheme (#27) --- docs/CONFIG-VARS.md | 2 +- roles/systems/postgres/tasks/main.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/CONFIG-VARS.md b/docs/CONFIG-VARS.md index 4437e1f..cabd112 100644 --- a/docs/CONFIG-VARS.md +++ b/docs/CONFIG-VARS.md @@ -261,7 +261,7 @@ nfs_ip = "" # Assigned values for static IP addresses **NOTES**: -1. If you set `server_ssl` to on, and you do not define either `server_ssl_cert_file` or `server_ssl_cert_file` the system's default SSL certificate and key will be used instead. By default, on Ubuntu systems we create a copy of those files and name them `ssl-cert-sas.pem` and `ssl-cert-sas.key` +1. If you set `server_ssl` to on, and you do not define either `server_ssl_cert_file` or `server_ssl_cert_file` the system's default SSL certificate and key will be used instead. By default, on Ubuntu systems we create a copy of those files and name them `ssl-cert-sas-${PG_HOST}.pem` and `ssl-cert-sas-${PG_HOST}.key` * The Ansible tasks will take care of copying the certificate and key from the Postgres VM into your local workspace directory 2. If you are planning on using the [viya4-deployment repository](https://github.com/sassoftware/viya4-deployment) to perform a Viya deployment where you have [full-stack TLS](https://github.com/sassoftware/viya4-deployment/blob/main/docs/CONFIG-VARS.md#tls), ensure in the viya4-deployment ansible-vars.yaml the `V4_CFG_TLS_TRUSTED_CA_CERTS` variable points to a directory that contains the `server_ssl_cert_file`. diff --git a/roles/systems/postgres/tasks/main.yaml b/roles/systems/postgres/tasks/main.yaml index c3587ff..f5d815f 100644 --- a/roles/systems/postgres/tasks/main.yaml +++ b/roles/systems/postgres/tasks/main.yaml @@ -101,7 +101,7 @@ - name: Copy Ubuntu default SSL Cert ansible.builtin.copy: src: /etc/ssl/certs/ssl-cert-snakeoil.pem - dest: /etc/ssl/certs/ssl-cert-sas.pem + dest: "/etc/ssl/certs/ssl-cert-sas-{{ ansible_hostname }}.pem" owner: root group: root mode: "0644" @@ -110,7 +110,7 @@ - name: Copy Ubuntu default SSL Key ansible.builtin.copy: src: /etc/ssl/private/ssl-cert-snakeoil.key - dest: /etc/ssl/private/ssl-cert-sas.key + dest: "/etc/ssl/private/ssl-cert-sas-{{ ansible_hostname }}.key" owner: root group: ssl-cert mode: "0640"