From e5beb0d8b71749baeb4ff846836a1637cce4b7c3 Mon Sep 17 00:00:00 2001 From: Jonatas Ferreira Date: Mon, 6 Nov 2023 20:12:56 -0300 Subject: [PATCH] change!: using kubespray to configure cluster Signed-off-by: Jonatas Ferreira --- .gitmodules | 3 + Makefile | 6 +- README.md | 10 +- Vagrantfile | 2 +- ansible/cluster.yml | 20 -- ansible/hosts.ini | 9 - ansible/reset.yml | 7 - ansible/roles/common/handlers/main.yml | 18 -- ansible/roles/common/tasks/main.yml | 52 ---- .../templates/containerd.config.toml.j2 | 250 ------------------ .../common/templates/k8s_modules.conf.j2 | 2 - .../roles/common/templates/k8s_sysctl.conf.j2 | 3 - ansible/roles/control-plane/tasks/main.yml | 49 ---- .../control-plane/templates/init_config.j2 | 9 - ansible/roles/data-plane/tasks/main.yml | 3 - create_hosts.sh | 38 ++- kubespray | 1 + 17 files changed, 47 insertions(+), 435 deletions(-) create mode 100644 .gitmodules delete mode 100644 ansible/cluster.yml delete mode 100644 ansible/hosts.ini delete mode 100644 ansible/reset.yml delete mode 100644 ansible/roles/common/handlers/main.yml delete mode 100644 ansible/roles/common/tasks/main.yml delete mode 100644 ansible/roles/common/templates/containerd.config.toml.j2 delete mode 100644 ansible/roles/common/templates/k8s_modules.conf.j2 delete mode 100644 ansible/roles/common/templates/k8s_sysctl.conf.j2 delete mode 100644 ansible/roles/control-plane/tasks/main.yml delete mode 100644 ansible/roles/control-plane/templates/init_config.j2 delete mode 100644 ansible/roles/data-plane/tasks/main.yml create mode 160000 kubespray diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..3264297 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "kubespray"] + path = kubespray + url = git@github.com:kubernetes-sigs/kubespray.git diff --git a/Makefile b/Makefile index b8892bd..346ae19 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,10 @@ SHELL := /bin/bash cluster: - vagrant up && bash create_hosts.sh && ansible-playbook -i ansible/hosts.ini ansible/cluster.yml + vagrant up && bash create_hosts.sh && cd kubespray && ansible-playbook -b -v -i inventory/vagrant/inventory.ini cluster.yml reset: - ansible-playbook -i ansible/hosts.ini ansible/reset.yml + cd kubespray && ansible-playbook -i inventory/vagrant/inventory.ini -b -v reset.yml destroy: - vagrant destroy -f + vagrant destroy -f && rm -rf kubespray/inventory/vagrant diff --git a/README.md b/README.md index e40b609..cda8b0d 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ Why? **Doing this way is more fun and more similar to how it is done in on-premi Of course this way it's more complicated, but it will be worth it. +Also, it is used [Kubespray](https://github.com/kubernetes-sigs/kubespray) to configure the production-ready K8S cluster. + ## Before we start ### Pre requisites @@ -54,7 +56,13 @@ sudo yum -y install vagrant ## Managing your cluster -The created cluster will use 6GB of RAM, so it's very likely that your machine will slow down a little. +First of all, clone this repository: + +```sh +git clone --recurse-submodules +``` + +The created cluster will use *GB of RAM, so it's very likely that your machine will slow down a little. ### Creating diff --git a/Vagrantfile b/Vagrantfile index d2baa60..db31960 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -14,7 +14,7 @@ Vagrant.configure("2") do |config| k8s.vm.provider "virtualbox" do |vb| vb.gui = false vb.cpus = 2 - vb.memory = "2048" + vb.memory = "4096" end end end diff --git a/ansible/cluster.yml b/ansible/cluster.yml deleted file mode 100644 index c48e618..0000000 --- a/ansible/cluster.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -- name: Install K8S on cluster - hosts: all - become: true - roles: - - common - -- name: Initialize cluster on control-plane - hosts: control-plane - become: true - gather_facts: false - roles: - - control-plane - -- name: Join worker nodes to control plane node - hosts: data-plane - become: true - gather_facts: false - roles: - - data-plane diff --git a/ansible/hosts.ini b/ansible/hosts.ini deleted file mode 100644 index 5d80b29..0000000 --- a/ansible/hosts.ini +++ /dev/null @@ -1,9 +0,0 @@ -[control-plane] -control-plane ansible_host=192.168.56.8 ansible_user=vagrant ansible_ssh_common_args='-o StrictHostKeyChecking=no' - -[control-plane:vars] -apiserver_advertise_address=192.168.56.8 - -[data-plane] -data-plane-1 ansible_host=192.168.56.9 ansible_user=vagrant ansible_ssh_common_args='-o StrictHostKeyChecking=no' -data-plane-2 ansible_host=192.168.56.10 ansible_user=vagrant ansible_ssh_common_args='-o StrictHostKeyChecking=no' diff --git a/ansible/reset.yml b/ansible/reset.yml deleted file mode 100644 index ca51cbd..0000000 --- a/ansible/reset.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: reset cluster - hosts: all - become: true - tasks: - - name: kubeadm reset - shell: kubeadm reset -f && rm -rf /etc/cni/net.d && rm -rf "/home/{{ ansible_user }}/.kube" diff --git a/ansible/roles/common/handlers/main.yml b/ansible/roles/common/handlers/main.yml deleted file mode 100644 index 514d099..0000000 --- a/ansible/roles/common/handlers/main.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -- name: Load overlay kernel module - modprobe: - name: overlay - state: present - -- name: Load br_netfilter kernel module - modprobe: - name: br_netfilter - state: present - -- name: Reload system modules - ansible.builtin.shell: sysctl --system - -- name: Restart containerd - ansible.builtin.systemd: - name: containerd - state: restarted diff --git a/ansible/roles/common/tasks/main.yml b/ansible/roles/common/tasks/main.yml deleted file mode 100644 index 74198bb..0000000 --- a/ansible/roles/common/tasks/main.yml +++ /dev/null @@ -1,52 +0,0 @@ ---- -- name: Download and add GPG Key - shell: curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-archive-keyring.gpg - -- name: Add K8S repository to /etc/apt/sources.list.d/kubernetes.list - apt_repository: - repo: deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main - state: present - filename: kubernetes.list - -- name: Update apt repository and install kubernetes and its dependencies - apt: - pkg: - - kubectl - - kubeadm - - kubelet - - apt-transport-https - - ca-certificates - - git - - golang-go - - containerd - update_cache: true - -- name: Mkdir /etc/containerd - file: - path: /etc/containerd - state: directory - -- name: Config containerd - template: - src: containerd.config.toml.j2 - dest: /etc/containerd/config.toml - notify: - - Restart containerd - -- name: Config K8s modules - template: - src: k8s_modules.conf.j2 - dest: /etc/modules-load.d/k8s.conf - notify: - - Load overlay kernel module - - Load br_netfilter kernel module - -- name: Config network modules - template: - src: k8s_sysctl.conf.j2 - dest: /etc/sysctl.d/k8s.conf - notify: - - Reload system modules - -- name: Turn off the swap - shell: swapoff -a diff --git a/ansible/roles/common/templates/containerd.config.toml.j2 b/ansible/roles/common/templates/containerd.config.toml.j2 deleted file mode 100644 index 31afe81..0000000 --- a/ansible/roles/common/templates/containerd.config.toml.j2 +++ /dev/null @@ -1,250 +0,0 @@ -disabled_plugins = [] -imports = [] -oom_score = 0 -plugin_dir = "" -required_plugins = [] -root = "/var/lib/containerd" -state = "/run/containerd" -temp = "" -version = 2 - -[cgroup] - path = "" - -[debug] - address = "" - format = "" - gid = 0 - level = "" - uid = 0 - -[grpc] - address = "/run/containerd/containerd.sock" - gid = 0 - max_recv_message_size = 16777216 - max_send_message_size = 16777216 - tcp_address = "" - tcp_tls_ca = "" - tcp_tls_cert = "" - tcp_tls_key = "" - uid = 0 - -[metrics] - address = "" - grpc_histogram = false - -[plugins] - - [plugins."io.containerd.gc.v1.scheduler"] - deletion_threshold = 0 - mutation_threshold = 100 - pause_threshold = 0.02 - schedule_delay = "0s" - startup_delay = "100ms" - - [plugins."io.containerd.grpc.v1.cri"] - device_ownership_from_security_context = false - disable_apparmor = false - disable_cgroup = false - disable_hugetlb_controller = true - disable_proc_mount = false - disable_tcp_service = true - enable_selinux = false - enable_tls_streaming = false - enable_unprivileged_icmp = false - enable_unprivileged_ports = false - ignore_image_defined_volumes = false - max_concurrent_downloads = 3 - max_container_log_line_size = 16384 - netns_mounts_under_state_dir = false - restrict_oom_score_adj = false - sandbox_image = "registry.k8s.io/pause:3.9" - selinux_category_range = 1024 - stats_collect_period = 10 - stream_idle_timeout = "4h0m0s" - stream_server_address = "127.0.0.1" - stream_server_port = "0" - systemd_cgroup = false - tolerate_missing_hugetlb_controller = true - unset_seccomp_profile = "" - - [plugins."io.containerd.grpc.v1.cri".cni] - bin_dir = "/opt/cni/bin" - conf_dir = "/etc/cni/net.d" - conf_template = "" - ip_pref = "" - max_conf_num = 1 - - [plugins."io.containerd.grpc.v1.cri".containerd] - default_runtime_name = "runc" - disable_snapshot_annotations = true - discard_unpacked_layers = false - ignore_rdt_not_enabled_errors = false - no_pivot = false - snapshotter = "overlayfs" - - [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime] - base_runtime_spec = "" - cni_conf_dir = "" - cni_max_conf_num = 0 - container_annotations = [] - pod_annotations = [] - privileged_without_host_devices = false - runtime_engine = "" - runtime_path = "" - runtime_root = "" - runtime_type = "" - - [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime.options] - - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - base_runtime_spec = "" - cni_conf_dir = "" - cni_max_conf_num = 0 - container_annotations = [] - pod_annotations = [] - privileged_without_host_devices = false - runtime_engine = "" - runtime_path = "" - runtime_root = "" - runtime_type = "io.containerd.runc.v2" - - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - BinaryName = "" - CriuImagePath = "" - CriuPath = "" - CriuWorkPath = "" - IoGid = 0 - IoUid = 0 - NoNewKeyring = false - NoPivotRoot = false - Root = "" - ShimCgroup = "" - SystemdCgroup = true - - [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime] - base_runtime_spec = "" - cni_conf_dir = "" - cni_max_conf_num = 0 - container_annotations = [] - pod_annotations = [] - privileged_without_host_devices = false - runtime_engine = "" - runtime_path = "" - runtime_root = "" - runtime_type = "" - - [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime.options] - - [plugins."io.containerd.grpc.v1.cri".image_decryption] - key_model = "node" - - [plugins."io.containerd.grpc.v1.cri".registry] - config_path = "" - - [plugins."io.containerd.grpc.v1.cri".registry.auths] - - [plugins."io.containerd.grpc.v1.cri".registry.configs] - - [plugins."io.containerd.grpc.v1.cri".registry.headers] - - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - - [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming] - tls_cert_file = "" - tls_key_file = "" - - [plugins."io.containerd.internal.v1.opt"] - path = "/opt/containerd" - - [plugins."io.containerd.internal.v1.restart"] - interval = "10s" - - [plugins."io.containerd.internal.v1.tracing"] - sampling_ratio = 1.0 - service_name = "containerd" - - [plugins."io.containerd.metadata.v1.bolt"] - content_sharing_policy = "shared" - - [plugins."io.containerd.monitor.v1.cgroups"] - no_prometheus = false - - [plugins."io.containerd.runtime.v1.linux"] - no_shim = false - runtime = "runc" - runtime_root = "" - shim = "containerd-shim" - shim_debug = false - - [plugins."io.containerd.runtime.v2.task"] - platforms = ["linux/amd64"] - sched_core = false - - [plugins."io.containerd.service.v1.diff-service"] - default = ["walking"] - - [plugins."io.containerd.service.v1.tasks-service"] - rdt_config_file = "" - - [plugins."io.containerd.snapshotter.v1.aufs"] - root_path = "" - - [plugins."io.containerd.snapshotter.v1.btrfs"] - root_path = "" - - [plugins."io.containerd.snapshotter.v1.devmapper"] - async_remove = false - base_image_size = "" - discard_blocks = false - fs_options = "" - fs_type = "" - pool_name = "" - root_path = "" - - [plugins."io.containerd.snapshotter.v1.native"] - root_path = "" - - [plugins."io.containerd.snapshotter.v1.overlayfs"] - root_path = "" - upperdir_label = false - - [plugins."io.containerd.snapshotter.v1.zfs"] - root_path = "" - - [plugins."io.containerd.tracing.processor.v1.otlp"] - endpoint = "" - insecure = false - protocol = "" - -[proxy_plugins] - -[stream_processors] - - [stream_processors."io.containerd.ocicrypt.decoder.v1.tar"] - accepts = ["application/vnd.oci.image.layer.v1.tar+encrypted"] - args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"] - env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"] - path = "ctd-decoder" - returns = "application/vnd.oci.image.layer.v1.tar" - - [stream_processors."io.containerd.ocicrypt.decoder.v1.tar.gzip"] - accepts = ["application/vnd.oci.image.layer.v1.tar+gzip+encrypted"] - args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"] - env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"] - path = "ctd-decoder" - returns = "application/vnd.oci.image.layer.v1.tar+gzip" - -[timeouts] - "io.containerd.timeout.bolt.open" = "0s" - "io.containerd.timeout.shim.cleanup" = "5s" - "io.containerd.timeout.shim.load" = "5s" - "io.containerd.timeout.shim.shutdown" = "3s" - "io.containerd.timeout.task.state" = "2s" - -[ttrpc] - address = "" - gid = 0 - uid = 0 diff --git a/ansible/roles/common/templates/k8s_modules.conf.j2 b/ansible/roles/common/templates/k8s_modules.conf.j2 deleted file mode 100644 index a66efc6..0000000 --- a/ansible/roles/common/templates/k8s_modules.conf.j2 +++ /dev/null @@ -1,2 +0,0 @@ -overlay -br_netfilter \ No newline at end of file diff --git a/ansible/roles/common/templates/k8s_sysctl.conf.j2 b/ansible/roles/common/templates/k8s_sysctl.conf.j2 deleted file mode 100644 index 107b4b0..0000000 --- a/ansible/roles/common/templates/k8s_sysctl.conf.j2 +++ /dev/null @@ -1,3 +0,0 @@ -net.bridge.bridge-nf-call-iptables = 1 -net.bridge.bridge-nf-call-ip6tables = 1 -net.ipv4.ip_forward = 1 \ No newline at end of file diff --git a/ansible/roles/control-plane/tasks/main.yml b/ansible/roles/control-plane/tasks/main.yml deleted file mode 100644 index 8a18038..0000000 --- a/ansible/roles/control-plane/tasks/main.yml +++ /dev/null @@ -1,49 +0,0 @@ ---- -- name: Send init config to control-plane - ansible.builtin.template: - src: init_config.j2 - dest: /tmp/init_config.yml - -- name: Init cluster with kubeadm - ansible.builtin.shell: | - kubeadm init --config /tmp/init_config.yml - -- name: Create .kube directory - ansible.builtin.file: - path: /home/{{ ansible_user }}/.kube - state: directory - mode: '0755' - owner: "{{ ansible_user }}" - group: "{{ ansible_user }}" - -- name: Create alias 'k' for kubectl - ansible.builtin.shell: | - echo 'source <(kubectl completion bash)' >> /home/{{ ansible_user }}/.bashrc - echo 'alias k=kubectl' >> /home/{{ ansible_user }}/.bashrc - echo 'alias kgp="kubectl get pods"'>> /home/{{ ansible_user }}/.bashrc - echo 'alias kgd="kubectl get deployments"'>> /home/{{ ansible_user }}/.bashrc - echo 'alias kgs="kubectl get svc"' >> /home/{{ ansible_user }}/.bashrc - echo 'alias kgn="kubectl get no"' >> /home/{{ ansible_user }}/.bashrc - echo 'complete -o default -F __start_kubectl k' >> /home/{{ ansible_user }}/.bashrc - -- name: Copy admin.conf from /etc/kubernetes to .kube - ansible.builtin.copy: - remote_src: true - src: /etc/kubernetes/admin.conf - dest: /home/{{ ansible_user }}/.kube/config - owner: "{{ ansible_user }}" - group: "{{ ansible_user }}" - -- name: Print join command to worker nodes - ansible.builtin.shell: echo "$(kubeadm token create --print-join-command)" | tee /tmp/join_command.sh - -- name: Fetching /tmp/join_command.sh to data-plane/files - ansible.builtin.fetch: - src: /tmp/join_command.sh - dest: "{{ playbook_dir }}/roles/data-plane/files/" - flat: true - -- name: Config pod network plugin - become: false - ansible.builtin.shell: | - kubectl apply -f https://projectcalico.docs.tigera.io/manifests/calico.yaml diff --git a/ansible/roles/control-plane/templates/init_config.j2 b/ansible/roles/control-plane/templates/init_config.j2 deleted file mode 100644 index 30ca3c2..0000000 --- a/ansible/roles/control-plane/templates/init_config.j2 +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: kubeadm.k8s.io/v1beta3 -kind: InitConfiguration -localAPIEndpoint: - advertiseAddress: "{{ apiserver_advertise_address }}" ---- -apiVersion: kubeadm.k8s.io/v1beta3 -kind: ClusterConfiguration -networking: - podSubnet: "10.244.0.0/16" diff --git a/ansible/roles/data-plane/tasks/main.yml b/ansible/roles/data-plane/tasks/main.yml deleted file mode 100644 index dafb816..0000000 --- a/ansible/roles/data-plane/tasks/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -- name: Join control-plane K8S cluster - ansible.builtin.script: join_command.sh diff --git a/create_hosts.sh b/create_hosts.sh index 16e0e96..b638132 100644 --- a/create_hosts.sh +++ b/create_hosts.sh @@ -1,5 +1,7 @@ #!/bin/bash +KUBESPRAY_PATH="./kubespray" + function get_public_ip() { vm=$1 vagrant ssh "$vm" -c 'hostname -I | cut -d " " -f2' | sed 's/\r$//' @@ -9,14 +11,34 @@ control_plane_ip="$(get_public_ip "control-plane-1")" data_plane_1_ip="$(get_public_ip "data-plane-1")" data_plane_2_ip="$(get_public_ip "data-plane-2")" -cat < ansible/hosts.ini -[control-plane] -$control_plane_ip ansible_user=vagrant ansible_ssh_common_args='-o StrictHostKeyChecking=no' +cluster_uuid="$KUBESPRAY_PATH/inventory/vagrant" + +cp -rfp "$KUBESPRAY_PATH/inventory/sample" "$cluster_uuid" + +cat < "$cluster_uuid/inventory.ini" +[all] +control-plane-1 ansible_host=$control_plane_ip ip=$control_plane_ip etcd_member_name=etcd1 +data-plane-1 ansible_host=$data_plane_1_ip ip=$data_plane_1_ip +data-plane-2 ansible_host=$data_plane_2_ip ip=$data_plane_2_ip + +[all:vars] +ansible_user=vagrant +ansible_ssh_common_args='-o StrictHostKeyChecking=no' + +[kube_control_plane] +control-plane-1 + +[etcd] +control-plane-1 + +[kube_node] +data-plane-1 +data-plane-2 -[control-plane:vars] -apiserver_advertise_address=$control_plane_ip +[calico_rr] -[data-plane] -$data_plane_1_ip ansible_user=vagrant ansible_ssh_common_args='-o StrictHostKeyChecking=no' -$data_plane_2_ip ansible_user=vagrant ansible_ssh_common_args='-o StrictHostKeyChecking=no' +[k8s_cluster:children] +kube_control_plane +kube_node +calico_rr EOF diff --git a/kubespray b/kubespray new file mode 160000 index 0000000..f3332af --- /dev/null +++ b/kubespray @@ -0,0 +1 @@ +Subproject commit f3332af3f2f26ecc46c9b88a538e70bf9225fb80