Fix/cleanup pre commit findings

ansible/group_vars/cluster.yml

@@ -0,0 +1,85 @@
+---
+
+# Specify the Kubernetes version, current release is v1.20.0
+cluster_kubernetes_version: 'v1.20.2'
+
+cluster_apiserver_enable_admission_plugins:
+- 'NodeRestriction'
+
+cluster_cgroup_driver: 'systemd'
+
+# Profiling; Enable profiling via web interface host:port/debug/pprof/
+cluster_profiling: false
+
+# The below parameters set the internal network for Kubernetes, this configuration
+# provides a maximum of 64 nodes in a cluster and 254 pods per node.
+
+# Kubernetes internal network for services, *must* be an unused block of space.
+cluster_service_subnet: '10.144.0.0/18'
+
+# Kubernetes internal network. IP's for pods will be drawn from this address
+# space. *must* be unused in your network infrastructure.
+cluster_pod_subnet: '10.144.64.0/18'
+
+# This sets the network node allocation size. Each node in the cluster is
+# assigned a subnet of this size.
+cluster_node_network_prefix: 24
+
+# Cluster Authorization Modes for the Kubernetes Cluster
+# Available Options: 'AlwaysAllow','AlwaysDeny','Node' and 'RBAC'
+# Order is relevant
+cluster_authorization_modes: ['Node', 'RBAC']
+
+# Vars for discoverying kubernetes api endpoints
+# will refactor to eliminate this variable
+cluster_primary_control_node: '{{ groups["masters"]|first }}'
+cluster_apiserver_count: '{{ groups["masters"]|length }}'
+
+# Tuning for arm64 platforms; https://etcd.io/docs/v3.4.0/tuning/
+cluster_etcd_heartbeat_interval: 250
+cluster_etcd_election_timeout: 1250
+cluster_etcd_snapshot_count: 5000
+
+####
+# Role - cri
+####
+cri_plugin: 'containerd'
+cri_containerd_version: '1.4.3'
+
+## Old variables
+####
+# Role - keepalived
+####
+keepalived_vip: 192.168.91.240
+keepalived_interface: "{{ ansible_default_ipv4['interface'] }}"
+
+####
+# Role - cluster
+####
+cluster_name: kubernetes
+cluster_extra_sans:
+- '{{ keepalived_vip }}'
+cluster_control_plane_endpoint: '{{ keepalived_vip }}:8443'
+
+####
+# Role - cni
+####
+cni_plugin: 'calico'
+# cni_bgp_peer_address: 192.168.0.1
+# cni_bgp_peer_asn: 64512
+# cni_cilium_helm_version: 1.9.3
+# cni_cilium_image_version: v1.9.3
+# cni_cilium_hubble_enabled: false
+# cni_cilium_enovy_proxy_image_version: v1.16.2
+# cni_cilium_bpf_hostrouting: "true"
+# cni_cilium_bpf_tproxy: "false"
+# cni_cilium_bpf_masquerade: "true"
+# cni_cilium_endpoint_routes: "false"
+
+####
+# Role - cloudflared
+####
+# Cloudflare options for exposing Kubernetes services via HTTPS/SSH
+cloudflared_enabled: false
+# cloudflared_kube_api_server_dns: ''
+# cloudflared_version: 2020.11.11

ansible/roles/cluster/defaults/main.yml

@@ -1,17 +1,112 @@
 ---
-cluster_name: kubernetes
-cluster_extra_sans:
-  - "{{ keepalived_vip }}"
-cluster_control_plane_endpoint: "" # defaults to '{{ keepalived_vip }}:8443' when left empty
-cluster_enable_admission_plugins: "NodeRestriction"
-cluster_authorization_mode: "Node,RBAC"
-cluster_audit_log_path: "/var/log/kubernetes/audit.log"
-cluster_audit_log_maxage: "30"
-cluster_audit_log_maxbackup: "10"
-cluster_audit_log_maxsize: "100"
-cluster_pod_subnet: ""
-# Default etcd values, change these if you experience "leader changed" issues when running on a SD card
+# API Server Variables
+
+# API Server bind address, default listen to all requests
+cluster_apiserver_bind_address: '0.0.0.0'
+
+# API Server secure port, default listen to 6443
+cluster_apiserver_bind_port: 6443
+
+# API Server insecure port, by default this port is disabled (0)
+# This configuration item has been marked deprecated and will be removed in future.
+cluster_apiserver_insecure_port: 0
+
+# Enable cluster audit log
+cluster_apiserver_enable_audit: false
+# path to the audit log
+cluster_apiserver_audit_log_path: '/var/log/kubernetes/audit.log'
+# audit log max age in days
+cluster_apiserver_audit_log_maxage: 30
+# number of audit logs to keep as backup
+cluster_apiserver_audit_log_maxbackup: 1
+# audit log max log size
+cluster_apiserver_audit_log_maxsize: 100
+
+# Admission Plugins to enable within the cluster
+# Default Plugins: NamespaceLifecycle, LimitRanger, ServiceAccount, TaintNodesByCondition, Priority, DefaultTolerationSeconds,
+#                  DefaultStorageClass, StorageObjectInUseProtection, PersistentVolumeClaimResize, RuntimeClass, CertificateApproval,
+#                  CertificateSigning, CertificateSubjectRestriction, DefaultIngressClass, MutatingAdmissionWebhook, ValidatingAdmissionWebhook, ResourceQuota
+cluster_apiserver_enable_admission_plugins: []
+
+# Admission Plugins to disable
+cluster_apiserver_disable_admission_plugins: []
+
+# Enable controller manager and scheduler accessible by cluster
+# necessary for metrics and monitoring; Default is '127.0.0.1'
+cluster_controller_manager_bind_address: '0.0.0.0'
+cluster_scheduler_bind_address: '0.0.0.0'
+
+# Number of terminated pods that can exist before the terminated pod
+# garbage collector starts deleting terminated pods. If <= 0, the terminated
+# pod garbage collector is disabled. Default: 12500
+cluster_controller_manager_terminated_pod_gc_threshold: 12500
+
+# Time in milliseconds; Time should be adjusted for 0.5-1.5x RTT between nodes.
+# If heartbeat is too low, etcd will send unnecessary messages that increase CPU.
+# Defaults: _heartbeat_interval: 100, _election_timeout: 1000
 cluster_etcd_heartbeat_interval: 100
 cluster_etcd_election_timeout: 1000
+
+# Number of committed transactions to trigger a snapshot to disk.
+# Default: 100000
+cluster_etcd_snapshot_count: 100000
+
 # Default will install kube-proxy although some CNI plugins will recommend it be disabled
 cluster_kube_proxy_enabled: true
+
+# Set the TLS Minimum version, Default VersionTLS11
+# Possible values: 'VersionTLS10', 'VersionTLS11', 'VersionTLS12', 'VersionTLS13'
+# cluster_tls_min_version: ''
+
+# Set the TLS Cipher Suites, Default GO cipher suites are used
+# cluster_tls_cipher_suites:
+# - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA
+# - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
+# - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
+# - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA
+# - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+# - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
+# - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA
+# - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA
+# - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA
+# - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256
+# - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
+# - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA
+# - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+# - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305
+# - TLS_ECDHE_RSA_WITH_RC4_128_SHA
+# - TLS_RSA_WITH_3DES_EDE_CBC_SHA
+# - TLS_RSA_WITH_AES_128_CBC_SHA
+# - TLS_RSA_WITH_AES_128_CBC_SHA256
+# - TLS_RSA_WITH_AES_128_GCM_SHA256
+# - TLS_RSA_WITH_AES_256_CBC_SHA
+# - TLS_RSA_WITH_AES_256_GCM_SHA384
+# - TLS_RSA_WITH_RC4_128_SHA
+
+cri_socket_map:
+  crio: 'unix:///var/run/crio/crio.sock'
+  containerd: 'unix:///run/containerd/containerd.sock'
+  docker: 'unix:///var/run/dockershim.sock'
+cri_socket: '{{ cri_socket_map[cri_plugin] }}'
+
+architecture_map:
+  amd64: 'amd64'
+  x86_64: 'amd64'
+  arm6l: 'arm'
+  arm7l: 'arm'
+  aarch64: 'arm64'
+  32-bit: '386'
+  64-bit: 'amd64'
+host_architecture: '{{ architecture_map[ansible_architecture] }}'
+
+
+# Old defaults
+cluster_name: kubernetes
+cluster_extra_sans:
+  - "{{ keepalived_vip }}"
+cluster_control_plane_endpoint: "" # defaults to '{{ keepalived_vip }}:8443' when left empty
+
+# Old Vars
+cluster_main_master: "{{ groups['masters'][0] }}"
+cluster_etcd_datadir: /var/lib/etcd
+cluster_image_repository: k8s.gcr.io

ansible/roles/cluster/tasks/initialize.yml

@@ -22,20 +22,20 @@
 
 - name: generate kubeadm configuration
   template:
-    src: kubeadm-init.yml.j2
-    dest: /etc/kubernetes/kubeadm-init.yml
+    src: kubeadm-init.yaml.j2
+    dest: /etc/kubernetes/kubeadm-init.yaml
     mode: 0644
 
 - name: initialize cluster (with kube-proxy)
   command:
-    cmd: kubeadm init --config /etc/kubernetes/kubeadm-init.yml --upload-certs
+    cmd: kubeadm init --config /etc/kubernetes/kubeadm-init.yaml --upload-certs
     creates: /etc/kubernetes/admin.conf
   when:
     - cluster_kube_proxy_enabled
 
 - name: initialize cluster (without kube-proxy)
   command:
-    cmd: kubeadm init --skip-phases=addon/kube-proxy --config /etc/kubernetes/kubeadm-init.yml --upload-certs
+    cmd: kubeadm init --skip-phases=addon/kube-proxy --config /etc/kubernetes/kubeadm-init.yaml --upload-certs
     creates: /etc/kubernetes/admin.conf
   when:
     - cluster_kube_proxy_enabled == false

ansible/roles/cluster/tasks/join.yml

@@ -5,12 +5,18 @@
 
 - name: generate kubeadm join configuration
   template:
-    src: kubeadm-join.yml.j2
-    dest: /etc/kubernetes/kubeadm-join.yml
+    src: kubeadm-join.yaml.j2
+    dest: /etc/kubernetes/kubeadm-join.yaml
     mode: 0644
   when: not cluster_node_configured
 
+- name: 'check/wait for cluster apiserver to be available'
+  ansible.builtin.wait_for:
+    host: '{{ hostvars[groups["masters"]|first]["ansible_default_ipv4"]["address"] }}'
+    port: '{{ cluster_apiserver_bind_port }}'
+    timeout: 180
+
 - name: join node to cluster
   command:
-    cmd: kubeadm join --config /etc/kubernetes/kubeadm-join.yml
+    cmd: kubeadm join --config /etc/kubernetes/kubeadm-join.yaml
     creates: /etc/kubernetes/kubelet.conf

ansible/roles/cluster/templates/kubeadm-init.yaml.j2

@@ -0,0 +1,112 @@
+#jinja2:lstrip_blocks: True
+---
+apiVersion: kubeadm.k8s.io/v1beta2
+kind: InitConfiguration
+bootstrapTokens:
+- token: {{ kubeadm_join_token }}
+  ttl: 1h
+  groups:
+  - system:bootstrappers:kubeadm:default-node-token
+  usages:
+  - signing
+  - authentication
+localAPIEndpoint:
+  advertiseAddress: {{ ansible_default_ipv4.address|default(ansible_all_ipv4_addresses[0]) }}
+  bindPort: {{ cluster_apiserver_bind_port }}
+certificateKey: {{ kubeadm_certificate_key }}
+nodeRegistration:
+  {% if inventory_hostname in groups['masters'] and inventory_hostname not in groups['workers'] %}
+  taints:
+  - effect: NoSchedule
+    key: node-role.kubernetes.io/master
+  {% else %}
+  taints: []
+  {% endif %}
+  criSocket: {{ cri_socket }}
+---
+apiVersion: kubeadm.k8s.io/v1beta2
+kind: ClusterConfiguration
+clusterName: {{ cluster_name }}
+certificatesDir: /etc/kubernetes/pki
+apiServer:
+  extraArgs:
+    authorization-mode: {{ cluster_authorization_modes | join(',') }}
+    bind-address: {{ cluster_apiserver_bind_address }}
+    apiserver-count: "{{ cluster_apiserver_count }}"
+    insecure-port: "{{ cluster_apiserver_insecure_port }}"
+    profiling: "{{ cluster_profiling }}"
+    {% if cluster_apiserver_enable_admission_plugins|length > 0 %}
+    enable-admission-plugins: {{ cluster_apiserver_enable_admission_plugins | join(',') }}
+    {% endif %}
+    {% if cluster_apiserver_disable_admission_plugins|length > 0 %}
+    disable-admission-plugins: {{ cluster_apiserver_disable_admission_plugins | join(',') }}
+    {% endif %}
+    {% if cluster_apiserver_enable_audit %}
+    audit-log-path: "{{ cluster_audit_log_path }}"
+    audit-log-maxage: "{{ cluster_audit_log_maxage }}"
+    audit-log-maxbackup: "{{ cluster_audit_log_maxbackup }}"
+    audit-log-maxsize: "{{ cluster_audit_log_maxsize }}"
+    {% endif %}
+    request-timeout: "1m0s"
+    service-account-lookup: "true"
+    {% if cluster_tls_min_version is defined %}
+    tls-min-version: {{ cluster_tls_min_version }}
+    {% endif %}
+    {% if cluster_tls_cipher_suites is defined %}
+    tls-cipher-suites: {{ cluster_tls_cipher_suites | join(',') }}
+    {% endif %}
+  certSANs:
+  {% for san in cluster_extra_sans %}
+  - {{ san }}
+  {% endfor %}
+  timeoutForControlPlane: "4m0s"
+etcd:
+  local:
+    {% if cluster_extra_sans | length > 0 %}
+    serverCertSANs:
+      {% for san in cluster_extra_sans %}
+      - {{ san }}
+      {% endfor %}
+    {% endif %}
+    dataDir: {{ cluster_etcd_datadir }}
+    extraArgs:
+      client-cert-auth: "true"
+      peer-client-cert-auth: "true"
+      peer-auto-tls: "false"
+      heartbeat-interval: "{{ cluster_etcd_heartbeat_interval }}"
+      election-timeout: "{{ cluster_etcd_election_timeout }}"
+      snapshot-count: "{{ cluster_etcd_snapshot_count }}"
+imageRepository: {{ cluster_image_repository }}
+networking:
+  dnsDomain: cluster.local
+  serviceSubnet: {{ cluster_service_subnet }}
+  podSubnet: {{ cluster_pod_subnet }}
+kubernetesVersion: {{ cluster_kubernetes_version }}
+controlPlaneEndpoint: {{ cluster_control_plane_endpoint }}
+controllerManager:
+  extraArgs:
+    bind-address: {{ cluster_controller_manager_bind_address }}
+    terminated-pod-gc-threshold: "{{ cluster_controller_manager_terminated_pod_gc_threshold }}"
+    profiling: "{{ cluster_profiling }}"
+    use-service-account-credentials: "true"
+    feature-gates: "RotateKubeletServerCertificate=true"
+    {% if cluster_tls_min_version is defined %}
+    tls-min-version: {{ cluster_tls_min_version }}
+    {% endif %}
+    {% if cluster_tls_cipher_suites is defined %}
+    tls-cipher-suites: {{ cluster_tls_cipher_suites | join(',') }}
+    {% endif %}
+scheduler:
+  extraArgs:
+    bind-address: {{ cluster_scheduler_bind_address }}
+    profiling: "{{ cluster_profiling }}"
+    {% if cluster_tls_min_version is defined %}
+    tls-min-version: {{ cluster_tls_min_version }}
+    {% endif %}
+    {% if cluster_tls_cipher_suites is defined %}
+    tls-cipher-suites: {{ cluster_tls_cipher_suites | join(',') }}
+    {% endif %}
+---
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+cgroupDriver: {{ cluster_cgroup_driver }}