Add L2 Load Balancer (#16)

Co-authored-by: Fedor Batonogov <[email protected]>

.ansible-lint

@@ -1,6 +1,10 @@
 skip_list:
   - risky-file-permissions
   - var-naming[no-role-prefix]
+  - yaml[indentation]
+  - run-once[task]
+  - no-changed-when
+  - partial-become[task]
 
 exclude_paths:
   - ansible/roles/haproxy_static_pods/files/haproxy.yaml

ansible/inventory.yml

@@ -3,27 +3,19 @@ kubeadm:
     kubeadm_control_plane:
       hosts:
         kubeadm-cp-01:
-          ansible_host: 10.0.70.70
+          ansible_host: 10.0.75.81
         kubeadm-cp-02:
-          ansible_host: 10.0.70.78
+          ansible_host: 10.0.75.82
         kubeadm-cp-03:
-          ansible_host: 10.0.70.79
+          ansible_host: 10.0.75.83
     kubeadm_nodes:
       hosts:
         kubeadm-node-01:
-          ansible_host: 10.0.70.71
+          ansible_host: 10.0.75.84
         kubeadm-node-02:
-          ansible_host: 10.0.70.77
+          ansible_host: 10.0.75.85
         kubeadm-node-03:
-          ansible_host: 10.0.70.74
-  vars:
-    ansible_user: infra
-    ansible_port: 22
-
-test_hosts:
-  hosts:
-    kubeadm-cp-01:
-      ansible_host: 10.0.75.203
+          ansible_host: 10.0.75.86
   vars:
     ansible_user: infra
     ansible_port: 22

ansible/kubeadm.yml

@@ -1,44 +1,29 @@
-# Запустите сервисы как статические подсистемы
-# https://github.com/kubernetes/kubeadm/blob/main/docs/ha-considerations.md#option-2-run-the-services-as-static-pods
-- name: Настраиваю keepalived + haproxy
-  become: true
-  hosts:
-    - kubeadm_control_plane
-  roles:
-    - haproxy_static_pods
-
-- name: Разворачиваю kubernetes кластер
+# Подготовка к запуску Kubernetes кластера
+- name: Подготоваливаю узлы для kubernetes кластера
   become: true
   hosts:
     - kubeadm
-  handlers:
-    - name: Перезагружаю виртуальные машины
-      ansible.builtin.reboot:
   tasks:
     - name: Добавляю модули br_netfilter и overlay
       community.general.modprobe:
-        name: '{{ item }}'
+        name: "{{ item }}"
         state: present
       with_items:
         - br_netfilter
         - overlay
-      # notify:
-      #   - Перезагружаю виртуальные машины
 
     - name: Добавляю модули br_netfilter и overlay в /etc/modules
       ansible.builtin.lineinfile:
         path: /etc/modules
-        line: '{{ item }}'
+        line: "{{ item }}"
         create: true
       with_items:
         - br_netfilter
         - overlay
-      # notify:
-      #   - Перезагружаю виртуальные машины
 
     - name: Включаю маршрутизацию IP и iptables для моста
       ansible.posix.sysctl:
-        name: '{{ item }}'
+        name: "{{ item }}"
         value: 1
         state: present
       with_items:
@@ -54,22 +39,31 @@
           - gpg
           - software-properties-common
         update_cache: true
+      register: apt_res
+      retries: 5
+      until: apt_res is success
 
     - name: Добавляю gpg ключ для репозиториев Kubernetes и cri-o
       ansible.builtin.apt_key:
         url: '{{ item["url"] }}'
         state: present
         keyring: '{{ item["keyring"] }}'
       with_items:
-        - {url: "https://pkgs.k8s.io/core:/stable:/v1.30/deb/Release.key", keyring: "/etc/apt/keyrings/kubernetes-apt-keyring.gpg"}
-        - {url: "https://pkgs.k8s.io/addons:/cri-o:/prerelease:/main/deb/Release.key", keyring: "/etc/apt/keyrings/cri-o-apt-keyring.gpg"}
+        - {
+            url: "https://pkgs.k8s.io/core:/stable:/v1.30/deb/Release.key",
+            keyring: "/etc/apt/keyrings/kubernetes-apt-keyring.gpg",
+          }
+        - {
+            url: "https://pkgs.k8s.io/addons:/cri-o:/prerelease:/main/deb/Release.key",
+            keyring: "/etc/apt/keyrings/cri-o-apt-keyring.gpg",
+          }
 
     - name: Добавляю репозитории Kubernetes и cri-o
       ansible.builtin.apt_repository:
-        repo: '{{ item }}'
+        repo: "{{ item }}"
         state: present
       with_items:
-        - deb [signed-by=/etc/apt/keyrings/cri-o-apt-keyring.gpg] https://pkgs.k8s.io/addons:/cri-o:/prerelease:/main/deb/ /
+        - deb [signed-by=/etc/apt/keyrings/cri-o-apt-keyring.gpg] https://pkgs.k8s.io/addons:/cri-o:/stable:/v1.30/deb/ /
         - deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.30/deb/ /
 
     - name: Устанавливаю пакеты kubelet, kubeadm, kubectl и cri-o
@@ -84,7 +78,7 @@
 
     - name: Предотвращаю обновление kubelet, kubeadm и kubectl
       ansible.builtin.dpkg_selections:
-        name: '{{ item }}'
+        name: "{{ item }}"
         selection: hold
       with_items:
         - kubelet
@@ -93,33 +87,88 @@
 
     - name: Включаю и запускаю службы kubelet и cri-o
       ansible.builtin.systemd:
-        name: '{{ item }}'
+        name: "{{ item }}"
         enabled: true
         state: started
       with_items:
         - kubelet
         - crio
 
-- name: Устанавливаю пакеты для Longhorn
+# Запуск сервисов keepalived и haproxy как статических подсистем
+# https://github.com/kubernetes/kubeadm/blob/main/docs/ha-considerations.md#option-2-run-the-services-as-static-pods
+- name: Настраиваю keepalived + haproxy и инициализирую кластер
   become: true
   hosts:
-    - kubeadm_nodes
+    - kubeadm_control_plane
+  roles:
+    - haproxy_static_pods
   tasks:
-    - name: Устанавливаю нужные пакеты
-      ansible.builtin.apt:
-        name:
-          - open-iscsi
-          - nfs-common
-          - bash
-          - curl
-          - grep
-        state: present
-        update_cache: true
+    - name: Инициализирую высокодоступный кластер
+      run_once: true
+      ansible.builtin.command: |
+        kubeadm init \
+          --pod-network-cidr=10.244.0.0/16 \
+          --control-plane-endpoint=10.0.75.80:8888 \
+          --upload-certs \
+          --skip-phases=addon/kube-proxy
+      args:
+        creates: /etc/kubernetes/kubelet.conf
+      notify:
+        - Создаю token для control-plane
+        - Создаю token для node
+        - Добавляю control-plane узлы в кластер
+        - Добавляю node узлы в кластер
+  handlers:
+    - name: Создаю token для control-plane
+      ansible.builtin.shell:
+        cmd: |
+          set -o pipefail
+          echo $(kubeadm token create --print-join-command) \
+            --control-plane \
+            --certificate-key \
+            $(kubeadm init phase upload-certs --upload-certs | grep -vw -e certificate -e Namespace)
+        executable: /bin/bash
+      register: join_control_plane_raw
+    - name: Создаю token для node
+      ansible.builtin.command: kubeadm token create --print-join-command
+      register: join_node_raw
+    - name: Добавляю control-plane узлы в кластер
+      ansible.builtin.command: "{{ join_control_plane_raw.stdout }}"
+      args:
+        creates: /etc/kubernetes/kubelet.conf
+      delegate_to: "{{ item }}"
+      loop: "{{ groups['kubeadm_control_plane'] }}"
+    - name: Добавляю node узлы в кластер
+      ansible.builtin.command: "{{ join_node_raw.stdout }}"
+      args:
+        creates: /etc/kubernetes/kubelet.conf
+      delegate_to: "{{ item }}"
+      loop: "{{ groups['kubeadm_nodes'] }}"
 
-    - name: Включаю и запускаю службы
-      ansible.builtin.systemd:
-        name: '{{ item }}'
-        enabled: true
-        state: started
-      with_items:
-        - iscsid
+# Подготовка control-plane узлов
+- name: Подготовка control-plane узлов для работы с kubectl
+  become: true
+  gather_facts: false
+  hosts:
+    - kubeadm_control_plane
+  tasks:
+    - name: Создаю директорию .kube
+      become_user: infra
+      ansible.builtin.file:
+        path: $HOME/.kube
+        state: directory
+        mode: "755"
+    - name: Копирую admin.conf в директорию .kube
+      ansible.builtin.copy:
+        src: /etc/kubernetes/admin.conf
+        dest: /home/infra/.kube/config
+        remote_src: true
+        owner: infra
+        group: infra
+        mode: "600"
+    - name: Копирую kube/config
+      run_once: true
+      ansible.posix.synchronize:
+        src: "~/.kube/config" # remote host
+        dest: "~/.kube/config" # localhost
+        mode: pull

ansible/roles/haproxy_static_pods/files/check_apiserver.sh

@@ -5,7 +5,7 @@ errorExit() {
     exit 1
 }
 
-curl --silent --max-time 2 --insecure https://localhost:7443/ -o /dev/null || errorExit "Error GET https://localhost:7443/"
-if ip addr | grep -q 10.0.70.85; then
-    curl --silent --max-time 2 --insecure https://10.0.70.85:7443/ -o /dev/null || errorExit "Error GET https://10.0.70.85:7443/"
+curl --silent --max-time 2 --insecure https://localhost:8888/ -o /dev/null || errorExit "Error GET https://localhost:8888/"
+if ip addr | grep -q 10.0.75.80; then
+    curl --silent --max-time 2 --insecure https://10.0.75.80:8888/ -o /dev/null || errorExit "Error GET https://10.0.75.80:8888/"
 fi

ansible/roles/haproxy_static_pods/files/haproxy.cfg

@@ -32,7 +32,7 @@ defaults
 # apiserver frontend which proxys to the control plane nodes
 #---------------------------------------------------------------------
 frontend apiserver
-    bind *:7443
+    bind *:8888
     mode tcp
     option tcplog
     default_backend apiserverbackend
@@ -46,7 +46,7 @@ backend apiserverbackend
     mode tcp
     option ssl-hello-chk
     balance     roundrobin
-        server 10.0.70.70 10.0.70.70:6443 check
-        server 10.0.70.78 10.0.70.78:6443 check
-        server 10.0.70.79 10.0.70.79:6443 check
+        server 10.0.75.81 10.0.75.81:6443 check
+        server 10.0.75.82 10.0.75.82:6443 check
+        server 10.0.75.83 10.0.75.83:6443 check
         # [...]

ansible/roles/haproxy_static_pods/files/haproxy.yaml

@@ -5,23 +5,23 @@ metadata:
   namespace: kube-system
 spec:
   containers:
-  - image: haproxy:2.9.7
-    name: haproxy
-    livenessProbe:
-      failureThreshold: 8
-      httpGet:
-        host: localhost
-        path: /healthz
-        port: 7443
-        scheme: HTTPS
-    volumeMounts:
-    - mountPath: /usr/local/etc/haproxy/haproxy.cfg
-      name: haproxyconf
-      readOnly: true
+    - image: haproxy:3.0.2
+      name: haproxy
+      livenessProbe:
+        failureThreshold: 8
+        httpGet:
+          host: localhost
+          path: /healthz
+          port: 8888
+          scheme: HTTPS
+      volumeMounts:
+        - mountPath: /usr/local/etc/haproxy/haproxy.cfg
+          name: haproxyconf
+          readOnly: true
   hostNetwork: true
   volumes:
-  - hostPath:
-      path: /etc/haproxy/haproxy.cfg
-      type: FileOrCreate
-    name: haproxyconf
+    - hostPath:
+        path: /etc/haproxy/haproxy.cfg
+        type: FileOrCreate
+      name: haproxyconf
 status: {}

ansible/roles/haproxy_static_pods/files/keepalived.yaml

@@ -6,26 +6,26 @@ metadata:
   namespace: kube-system
 spec:
   containers:
-  - image: osixia/keepalived:2.0.20
-    name: keepalived
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - NET_BROADCAST
-        - NET_RAW
-    volumeMounts:
-    - mountPath: /usr/local/etc/keepalived/keepalived.conf
-      name: config
-    - mountPath: /etc/keepalived/check_apiserver.sh
-      name: check
+    - image: ghcr.io/batonogov/docker-keepalived:2.0.20
+      name: keepalived
+      resources: {}
+      securityContext:
+        capabilities:
+          add:
+            - NET_ADMIN
+            - NET_BROADCAST
+            - NET_RAW
+      volumeMounts:
+        - mountPath: /usr/local/etc/keepalived/keepalived.conf
+          name: config
+        - mountPath: /etc/keepalived/check_apiserver.sh
+          name: check
   hostNetwork: true
   volumes:
-  - hostPath:
-      path: /etc/keepalived/keepalived.conf
-    name: config
-  - hostPath:
-      path: /etc/keepalived/check_apiserver.sh
-    name: check
+    - hostPath:
+        path: /etc/keepalived/keepalived.conf
+      name: config
+    - hostPath:
+        path: /etc/keepalived/check_apiserver.sh
+      name: check
 status: {}